Datapreparation-SumStats.R

#title: "Data Preparation & Summary Stats"
#author: "Qianhui Li"

setwd("/Users/qianhuili/Desktop/GitHub/AAE724/Script/Data_cleaning")

library(tidyr)   
library(dplyr)   

## 
## Attaching package: 'dplyr'

## The following objects are masked from 'package:stats':
## 
##     filter, lag

## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union

library(leaps)
library(glmnet)

## Loading required package: Matrix

## 
## Attaching package: 'Matrix'

## The following objects are masked from 'package:tidyr':
## 
##     expand, pack, unpack

## Loading required package: foreach

## Loaded glmnet 2.0-18

library(ggplot2)
library(gmodels)
library(MASS)

## 
## Attaching package: 'MASS'

## The following object is masked from 'package:dplyr':
## 
##     select

library(corrplot)

## corrplot 0.84 loaded

library(ISLR)
library(tree)
library(gridExtra)

## 
## Attaching package: 'gridExtra'

## The following object is masked from 'package:dplyr':
## 
##     combine

library(ROCR)

## Loading required package: gplots

## 
## Attaching package: 'gplots'

## The following object is masked from 'package:stats':
## 
##     lowess

library(rpart)
library(rpart.plot)
library(rattle)

## Rattle: A free graphical interface for data science with R.
## Version 5.2.0 Copyright (c) 2006-2018 Togaware Pty Ltd.
## Type 'rattle()' to shake, rattle, and roll your data.

library(pROC)

## Type 'citation("pROC")' for a citation.

## 
## Attaching package: 'pROC'

## The following object is masked from 'package:gmodels':
## 
##     ci

## The following object is masked from 'package:glmnet':
## 
##     auc

## The following objects are masked from 'package:stats':
## 
##     cov, smooth, var

library(corrplot)
library(lfe)
library(car)

## Loading required package: carData

## 
## Attaching package: 'car'

## The following object is masked from 'package:dplyr':
## 
##     recode

library(tidyverse)

## Registered S3 method overwritten by 'cli':
##   method     from
##   print.tree tree

## ── Attaching packages ─────────────────────────────────────────── tidyverse 1.2.1 ──

## ✔ tibble  2.1.3     ✔ purrr   0.3.2
## ✔ readr   1.3.1     ✔ stringr 1.4.0
## ✔ tibble  2.1.3     ✔ forcats 0.4.0

## ── Conflicts ────────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ purrr::accumulate()  masks foreach::accumulate()
## ✖ gridExtra::combine() masks dplyr::combine()
## ✖ Matrix::expand()     masks tidyr::expand()
## ✖ dplyr::filter()      masks stats::filter()
## ✖ dplyr::lag()         masks stats::lag()
## ✖ Matrix::pack()       masks tidyr::pack()
## ✖ car::recode()        masks dplyr::recode()
## ✖ MASS::select()       masks dplyr::select()
## ✖ purrr::some()        masks car::some()
## ✖ Matrix::unpack()     masks tidyr::unpack()
## ✖ purrr::when()        masks foreach::when()

library(viridis)

## Loading required package: viridisLite

library(ggpubr)

## Loading required package: magrittr

## 
## Attaching package: 'magrittr'

## The following object is masked from 'package:purrr':
## 
##     set_names

## The following object is masked from 'package:tidyr':
## 
##     extract

library(plotly)

## 
## Attaching package: 'plotly'

## The following object is masked from 'package:MASS':
## 
##     select

## The following object is masked from 'package:ggplot2':
## 
##     last_plot

## The following object is masked from 'package:stats':
## 
##     filter

## The following object is masked from 'package:graphics':
## 
##     layout

library(corrplot)
library(ROSE)

## Loaded ROSE 0.0-3

library(naniar)
library(caret)

## Loading required package: lattice

## 
## Attaching package: 'caret'

## The following object is masked from 'package:purrr':
## 
##     lift

library(blorr)
library(pROC)
#=============================================

##Data Preparation

bankoriginal<-read.csv("bank_data.csv",header=TRUE, sep=";", na.strings=c("unknown","non-existent"))

#Check # & % of missing values
gg_miss_var(bankoriginal)

gg_miss_var(bankoriginal, show_pct = TRUE)

#Since there is "999" in pdays means client was not previously contacted, I convert pdays into a dummy variable, never contacted(999)=0,others=1.

bankoriginal$pdays <-as.factor(bankoriginal$pdays)
bankoriginal$pdays <-ifelse(bankoriginal$pdays==999,0,1)


#The first variable that has the largest proportion of missing values is "default",
#However, it may be possible that customer is not willing to disclose this information to the banking representative. 
#Hence the unknown value in 'default' is actually a separate value.
#Thus I kept the variable "default", and I think it also make sense for "loan" and "housing" loan variable
bankoriginal$default <- as.character(bankoriginal$default)
bankoriginal$default[is.na(bankoriginal$default)] <- "refuse2disclose"

bankoriginal$loan<-as.character(bankoriginal$loan)
bankoriginal$loan[is.na(bankoriginal$loan)] <- "refuse2disclose"

bankoriginal$housing<-as.character(bankoriginal$housing)
bankoriginal$housing[is.na(bankoriginal$housing)] <- "refuse2disclose"
#As indicated by the data contributor, the duration is not known before a call is performed. 
#Also, after the end of the call y is obviously known. 
#Thus, this input should only be included for benchmark purposes and should be discarded if the intention is to have a realistic predictive model.
#Thus I removed "duration"
bankoriginal = bankoriginal %>% 
  select(-duration)

#check for missing value graph again
gg_miss_var(bankoriginal)

gg_miss_var(bankoriginal, show_pct = TRUE)

#omit missing values
bank<-na.omit(bankoriginal)
sum(is.na(bank))

## [1] 0

#Data summary
summary(bank)

##       age                 job            marital     
##  Min.   :17.00   admin.     :10159   divorced: 4417  
##  1st Qu.:32.00   blue-collar: 8788   married :23748  
##  Median :38.00   technician : 6520   single  :11026  
##  Mean   :39.86   services   : 3814                   
##  3rd Qu.:47.00   management : 2798                   
##  Max.   :98.00   retired    : 1617                   
##                  (Other)    : 5495                   
##                education       default            housing         
##  basic.4y           : 4118   Length:39191       Length:39191      
##  basic.6y           : 2264   Class :character   Class :character  
##  basic.9y           : 6006   Mode  :character   Mode  :character  
##  high.school        : 9464                                        
##  illiterate         :   18                                        
##  professional.course: 5225                                        
##  university.degree  :12096                                        
##      loan                contact          month       day_of_week
##  Length:39191       cellular :24983   may    :13128   fri:7417   
##  Class :character   telephone:14208   jul    : 6767   mon:8107   
##  Mode  :character                     aug    : 5947   thu:8194   
##                                       jun    : 5014   tue:7683   
##                                       nov    : 3973   wed:7790   
##                                       apr    : 2493              
##                                       (Other): 1869              
##     campaign          pdays           previous             poutcome    
##  Min.   : 1.000   Min.   :0.0000   Min.   :0.0000   failure    : 4044  
##  1st Qu.: 1.000   1st Qu.:0.0000   1st Qu.:0.0000   nonexistent:33877  
##  Median : 2.000   Median :0.0000   Median :0.0000   success    : 1270  
##  Mean   : 2.566   Mean   :0.0357   Mean   :0.1704                      
##  3rd Qu.: 3.000   3rd Qu.:0.0000   3rd Qu.:0.0000                      
##  Max.   :56.000   Max.   :1.0000   Max.   :7.0000                      
##                                                                        
##   emp.var.rate      cons.price.idx  cons.conf.idx      euribor3m    
##  Min.   :-3.40000   Min.   :92.20   Min.   :-50.80   Min.   :0.634  
##  1st Qu.:-1.80000   1st Qu.:93.08   1st Qu.:-42.70   1st Qu.:1.344  
##  Median : 1.10000   Median :93.44   Median :-41.80   Median :4.857  
##  Mean   : 0.08324   Mean   :93.57   Mean   :-40.54   Mean   :3.624  
##  3rd Qu.: 1.40000   3rd Qu.:93.99   3rd Qu.:-36.40   3rd Qu.:4.961  
##  Max.   : 1.40000   Max.   :94.77   Max.   :-26.90   Max.   :5.045  
##                                                                     
##   nr.employed     y        
##  Min.   :4964   no :34831  
##  1st Qu.:5099   yes: 4360  
##  Median :5191              
##  Mean   :5167              
##  3rd Qu.:5228              
##  Max.   :5228              
## 

#convert variable types
sapply(bank,class)

##            age            job        marital      education        default 
##      "integer"       "factor"       "factor"       "factor"    "character" 
##        housing           loan        contact          month    day_of_week 
##    "character"    "character"       "factor"       "factor"       "factor" 
##       campaign          pdays       previous       poutcome   emp.var.rate 
##      "integer"      "numeric"      "integer"       "factor"      "numeric" 
## cons.price.idx  cons.conf.idx      euribor3m    nr.employed              y 
##      "numeric"      "numeric"      "numeric"      "numeric"       "factor"

  #numerical variables
bank$age <- as.numeric(bank$age)
bank$campaign <- as.numeric(bank$campaign)
bank$previous <- as.numeric(bank$previous)
bank$emp.var.rate <- as.numeric(bank$emp.var.rate)
bank$cons.price.idx <- as.numeric(bank$cons.price.idx)
bank$cons.conf.idx <- as.numeric(bank$cons.conf.idx)
bank$euribor3m <- as.numeric(bank$euribor3m)
bank$nr.employed <- as.numeric(bank$nr.employed)

  #categorical variables
bank$job <-as.factor(bank$job)
bank$marital <-as.factor(bank$marital)
bank$education <-as.factor(bank$education)
bank$default <-as.factor(bank$default)
bank$loan <-as.factor(bank$loan)
bank$housing<-as.factor(bank$housing)
bank$contact <-as.factor(bank$contact)
bank$poutcome <-as.factor(bank$poutcome)
bank$day_of_week <-as.factor(bank$day_of_week)
bank$month <-as.factor(bank$month)

bank$y<-ifelse(bank$y =='yes',1,0)
bank$y <-as.factor(bank$y)

#Check for outliers for numerical variables
p1_age <- ggplot(bank, aes(y, age)) + geom_boxplot(aes(fill = y))

p1_campaign <- ggplot(bank, aes(y, campaign)) + geom_boxplot(aes(fill = y))

p1_previous <- ggplot(bank, aes(y, previous)) + geom_boxplot(aes(fill = y))

p1_emp.var.rate <- ggplot(bank, aes(y, emp.var.rate)) + geom_boxplot(aes(fill = y))

p1_cons.price.idx <- ggplot(bank, aes(y, cons.price.idx)) + geom_boxplot(aes(fill = y))

p1_cons.conf.idx<- ggplot(bank, aes(y, cons.conf.idx)) + geom_boxplot(aes(fill = y))

p1_euribor3m<- ggplot(bank, aes(y, euribor3m)) + geom_boxplot(aes(fill = y))

p1_nr.employed<- ggplot(bank, aes(y, nr.employed)) + geom_boxplot(aes(fill = y))


a1 <- c(p1_age,p1_campaign)
ggarrange(p1_age,p1_campaign, 
          nrow = 1)

b1 <- c(p1_previous,p1_emp.var.rate,
       p1_cons.price.idx)
ggarrange(p1_previous,p1_emp.var.rate,
          p1_cons.price.idx, 
          nrow = 1)

g1 <- c(p1_cons.conf.idx,
       p1_euribor3m,p1_nr.employed)
ggarrange(p1_cons.conf.idx,p1_euribor3m,p1_nr.employed, 
          nrow = 1)

#age==38770
x <- bank$age
qnt <- quantile(x, probs=c(.25, .75), na.rm = T)
H <- 1.5 * IQR(x, na.rm = T)
hb <- H + qnt[2]
hb #remove>69.5

##  75% 
## 69.5

ab <- bank[which(bank$age<hb),]

#campaign==35982
x1 <- bank$campaign
qnt1 <- quantile(x1, probs=c(.25, .75), na.rm = T)
H1 <- 1.5 * IQR(x1, na.rm = T)
hb1<- H1 + qnt1[2]
hb1 #remove>6

## 75% 
##   6

ac <- bank[which(bank$campaign<hb1),]

#cons.conf.idx
x5 <- bank$cons.conf.idx 
qnt5 <- quantile(x5, probs=c(.25, .75), na.rm = T)
H5 <- 1.5 * IQR(x5, na.rm = T)
hb5<- H5 + qnt5[2]
hb5 #remove>-26.95

##    75% 
## -26.95

#From the boxplot for "previous", I decided to treat observations larger than 2 as outliers, thus remove them.

#Result after removing outliers in numerical variables(34,370obs with 20 variables)
bank <- bank[which(bank$age<hb & bank$campaign<hb1 & bank$previous<2 & bank$cons.conf.idx<hb5),]


#Check for outliers for categorical variables with more than 3 categories

pic_job1 <-ggplot(bank, aes(x=job)) + geom_histogram(aes(y=(..count..)), stat='count', fill="slate blue", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Job",
       x="Job", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_job1

#From the histogram, there is no obvious small number of counts for jobs

pic_edu1 <-ggplot(bank, aes(x=education)) + geom_histogram(aes(y=(..count..)), stat='count', fill="yellowgreen", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Education",
       x="Education Status", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_edu1

table(bank$education)

## 
##            basic.4y            basic.6y            basic.9y 
##                3450                2022                5418 
##         high.school          illiterate professional.course 
##                8330                  16                4592 
##   university.degree 
##               10542

bank=bank[!as.factor(bank$education) %in% which(table(bank$education)=="illiterate"),]
bank$education=droplevels(bank$education)
#From the histogram, there is one obvious tiny number of counts for "illterate"(16 observations)
#Thus I decided to drop obs with "illiterate"

#After removing outliers for both numerical and categorical variables, there are 34,354 obs with 20 variables.

#Check and adjust data imbalance
counts <- table(bank$y)
barplot(counts,col=c("royalblue3","tomato3"),legend = rownames(counts), main = "Term Deposit")

CrossTable(bank$y)

## 
##  
##    Cell Contents
## |-------------------------|
## |                       N |
## |         N / Table Total |
## |-------------------------|
## 
##  
## Total Observations in Table:  34370 
## 
##  
##           |         0 |         1 | 
##           |-----------|-----------|
##           |     30946 |      3424 | 
##           |     0.900 |     0.100 | 
##           |-----------|-----------|
## 
## 
## 
## 

#From the graph and the table, we can see that the dataset is highly imbalanced
#Since most machine learning classification algorithms are sensitive to unbalance in the predictor classes.
#I decided to resample the data by Synthetic Minority Oversampling Technique (SMOTE),
#which is a popular algorithm to creates synthetic observations of the minority class

set.seed(888)

balanced_data <- ROSE(y~., data=bank,seed=1)$data
CrossTable(balanced_data$y)

## 
##  
##    Cell Contents
## |-------------------------|
## |                       N |
## |         N / Table Total |
## |-------------------------|
## 
##  
## Total Observations in Table:  34370 
## 
##  
##           |         0 |         1 | 
##           |-----------|-----------|
##           |     17253 |     17117 | 
##           |     0.502 |     0.498 | 
##           |-----------|-----------|
## 
## 
## 
## 

counts1 <- table(balanced_data$y)
barplot(counts1,col=c("royalblue3","tomato3"),legend = rownames(counts), main = "Customers' Responses")

#Hence, we now have 17245(50.2%) "no" responses, and 17109 (49.8%)"yes" responses, thus the data is balanced.

#=============================================

##Summary Statistics

  #categorical variables exploration
pic_job <-ggplot(balanced_data, aes(x=job)) + geom_histogram(aes(y=(..count..)), stat='count', fill="slate blue", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Job",
       x="Job", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_job

  #The graph shows that the there are alot of customers work in administritive sector, and the least as entrepreneur.

aa <-ggplot(balanced_data, aes(x = job , fill = y)) +
  geom_bar(stat='count', position='dodge')
aa

  #The graph shows that there are customers that are admin, retired, or technicial are more willing to accept the offer.
#\\\\\\
pic_marital <-ggplot(balanced_data, aes(x=marital)) + geom_histogram(aes(y=(..count..)), stat='count', fill="light pink", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Marital",
       x="Marital Status", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_marital

bb<-ggplot(balanced_data, aes(x = marital , fill = y)) +
  geom_bar(stat='count', position='dodge')
bb

#\\\\\\

pic_edu <-ggplot(balanced_data, aes(x=education)) + geom_histogram(aes(y=(..count..)), stat='count', fill="yellowgreen", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Education",
       x="Education Status", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_edu

cc<-ggplot(balanced_data, aes(x = education , fill = y)) +
  geom_bar(stat='count', position='dodge')
cc

#\\\\\\

pic_default <-ggplot(balanced_data, aes(x=default)) + geom_histogram(aes(y=(..count..)), stat='count', fill="light blue", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Default",
       x="Default Status", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_default

dd<-ggplot(balanced_data, aes(x = default , fill = y)) +
  geom_bar(stat='count', position='dodge')
dd

#\\\\\\

pic_loan <-ggplot(balanced_data, aes(x=loan)) + geom_histogram(aes(y=(..count..)), stat='count', fill="orange1", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Loan",
       x="Loan Status", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_loan

ee<-ggplot(balanced_data, aes(x = loan , fill = y)) +
  geom_bar(stat='count', position='dodge')
ee

#\\\\\\
pic_housing <-ggplot(balanced_data, aes(x=housing)) + geom_histogram(aes(y=(..count..)), stat='count', fill="grey69", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Housing",
       x="Housing Status", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_housing

ff<-ggplot(balanced_data, aes(x = housing , fill = y)) +
  geom_bar(stat='count', position='dodge')
ff

#\\\\\\
pic_contact <-ggplot(balanced_data, aes(x=contact)) + geom_histogram(aes(y=(..count..)), stat='count', fill="firebrick", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Contact",
       x="Contact Approach", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_contact

gg<-ggplot(balanced_data, aes(x = contact , fill = y)) +
  geom_bar(stat='count', position='dodge')
gg

#\\\\\\
pic_poutcome <-ggplot(balanced_data, aes(x=poutcome)) + geom_histogram(aes(y=(..count..)), stat='count', fill="yellow1", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "poutcome",
       x="Previous Outcome", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_poutcome

hh<-ggplot(balanced_data, aes(x = poutcome , fill = y)) +
  geom_bar(stat='count', position='dodge')
hh

#\\\\\\
pic_dow <-ggplot(balanced_data, aes(x=day_of_week)) + geom_histogram(aes(y=(..count..)), stat='count', fill="turquoise4", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Day of Week",
       x="Day of Week", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_dow

jj<-ggplot(balanced_data, aes(x = day_of_week , fill = y)) +
  geom_bar(stat='count', position='dodge')
jj

#\\\\\\
pic_month <-ggplot(balanced_data, aes(x=month)) + geom_histogram(aes(y=(..count..)), stat='count', fill="darkseagreen4", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Month",
       x="Months", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_month

kk<-ggplot(balanced_data, aes(x = month , fill = y)) +
  geom_bar(stat='count', position='dodge')
kk

#\\\\\\
#response variable
pic_y <-ggplot(balanced_data, aes(x=y)) + geom_histogram(aes(y=(..count..)), stat='count', fill="red", alpha=0.5) + theme_minimal() + 
  theme(plot.title    = element_text(face = "bold", size = 14, hjust = 0.5),
        axis.text.x     = element_text(angle = 45, hjust = 1, size=10),
        axis.text.y     = element_text(size=10)) +
  labs(title    = "Subscribe or not",
       x="Subscription", y="Counts")

## Warning: Ignoring unknown parameters: binwidth, bins, pad

pic_y

CrossTable(balanced_data$y)

## 
##  
##    Cell Contents
## |-------------------------|
## |                       N |
## |         N / Table Total |
## |-------------------------|
## 
##  
## Total Observations in Table:  34370 
## 
##  
##           |         0 |         1 | 
##           |-----------|-----------|
##           |     17253 |     17117 | 
##           |     0.502 |     0.498 | 
##           |-----------|-----------|
## 
## 
## 
## 

  #numerical variables exploration
p_age <- ggplot(balanced_data, aes(y, age)) + geom_boxplot(aes(fill = y))
hist(balanced_data$age, col = "yellow2", freq = FALSE)
abline(v = mean(balanced_data$age),
       col = "royalblue",
       lwd = 2)
abline(v = median(balanced_data$age),
       col = "light pink",
       lwd = 2)
legend(x = "topright", 
       c("Density plot", "Mean", "Median"),
       col = c("yellow2", "royalblue", "light pink"),
       lwd = c(2, 2, 2))

#The distribution shows that most customers oberved are less than 40 years old.


p_campaign <- ggplot(balanced_data, aes(y, campaign)) + geom_boxplot(aes(fill = y))

p_pdays <- ggplot(balanced_data, aes(y, pdays)) + geom_boxplot(aes(fill = y))
p_pdays

p_previous <- ggplot(balanced_data, aes(y, previous)) + geom_boxplot(aes(fill = y))
p_previous

p_emp.var.rate <- ggplot(balanced_data, aes(y, emp.var.rate)) + geom_boxplot(aes(fill = y))


p_cons.price.idx <- ggplot(balanced_data, aes(y, cons.price.idx)) + geom_boxplot(aes(fill = y))


p_cons.conf.idx<- ggplot(balanced_data, aes(y, cons.conf.idx)) + geom_boxplot(aes(fill = y))


p_euribor3m<- ggplot(balanced_data, aes(y, euribor3m)) + geom_boxplot(aes(fill = y))


p_nr.employed<- ggplot(balanced_data, aes(y, nr.employed)) + geom_boxplot(aes(fill = y))


a <- c(p_age,p_campaign,p_pdays)
ggarrange(p_age,p_campaign, 
          nrow = 1)

b <- c(p_previous,p_emp.var.rate,
       p_cons.price.idx)
ggarrange(p_previous,p_emp.var.rate,
          p_cons.price.idx, 
          nrow = 1)

g <- c(p_cons.conf.idx,
       p_euribor3m,p_nr.employed)
ggarrange(p_cons.conf.idx,p_euribor3m,p_nr.employed, 
          nrow = 1)

numericdata <- subset(balanced_data, select=c("age", "campaign","previous","emp.var.rate","cons.price.idx","cons.conf.idx","euribor3m","nr.employed","pdays","previous"))

pairs(numericdata)

M <- cor(numericdata)
corrplot(M, method = "circle")

#or view in corr magnitudes
corrplot(M, method = "number")

#From the correlation plot, we can see that there are good correlations between 'cons.price.idx'&'emp.var.rate', 'cons.conf.idx'&'emp.var.rate',cons.conf.idx'&'cons.price.idx','cons.price.idx'&'nr.employed', cons.conf.idx'&'nr.employed','emp.var.rate'& nr.employed',nr.employed'& euribor3m.
 #Those multicollinearity problems may not affect our predictions but indeed affect causal inferences.

#=============================================

#Data Split for Logistic Regression, CART, and Random Forest

set.seed(888)
index <- createDataPartition(balanced_data$y, p = 0.5, list = FALSE)
train_data <- balanced_data[index, ]
test_data  <- balanced_data[-index, ] 
#===========================================================
#Predictions

  #Logistic Regression

## Feature selection: For logistic regression and neural nets, use 13 variables
modelk <- glm(y ~ ., data = balanced_data, family = binomial(link = 'logit'))
blr_step_aic_both(modelk)

## Stepwise Selection Method 
## -------------------------
## 
## Candidate Terms: 
## 
## 1 . age 
## 2 . job 
## 3 . marital 
## 4 . education 
## 5 . default 
## 6 . housing 
## 7 . loan 
## 8 . contact 
## 9 . month 
## 10 . day_of_week 
## 11 . campaign 
## 12 . pdays 
## 13 . previous 
## 14 . poutcome 
## 15 . emp.var.rate 
## 16 . cons.price.idx 
## 17 . cons.conf.idx 
## 18 . euribor3m 
## 19 . nr.employed 
## 
## 
## Variables Entered/Removed: 
## 
## - nr.employed added 
## - month added 
## - poutcome added 
## - emp.var.rate added 
## - contact added 
## - cons.conf.idx added 
## - job added 
## - euribor3m added 
## - day_of_week added 
## - default added 
## - education added 
## - previous added 
## - campaign added 
## - housing added 
## - marital added 
## 
## No more variables to be added or removed.

## 
## 
##                          Stepwise Summary                         
## ----------------------------------------------------------------
## Variable          Method        AIC          BIC       Deviance  
## ----------------------------------------------------------------
## nr.employed      addition    42303.598    42320.488    42299.598 
## month            addition    40550.587    40643.482    40528.587 
## poutcome         addition    39938.667    40048.451    39912.667 
## emp.var.rate     addition    39454.351    39572.581    39426.351 
## contact          addition    39315.247    39441.921    39285.247 
## cons.conf.idx    addition    39232.783    39367.902    39200.783 
## job              addition    39160.690    39380.258    39108.690 
## euribor3m        addition    39096.318    39324.331    39042.318 
## day_of_week      addition    39036.811    39298.604    38974.811 
## default          addition    38994.654    39273.337    38928.654 
## education        addition    38963.255    39292.608    38885.255 
## previous         addition    38959.217    39297.014    38879.217 
## campaign         addition    38955.568    39301.811    38873.568 
## housing          addition    38951.955    39315.087    38865.955 
## marital          addition    38950.828    39330.850    38860.828 
## ----------------------------------------------------------------

modelk %>% blr_step_aic_both() %>%plot()

## Stepwise Selection Method 
## -------------------------
## 
## Candidate Terms: 
## 
## 1 . age 
## 2 . job 
## 3 . marital 
## 4 . education 
## 5 . default 
## 6 . housing 
## 7 . loan 
## 8 . contact 
## 9 . month 
## 10 . day_of_week 
## 11 . campaign 
## 12 . pdays 
## 13 . previous 
## 14 . poutcome 
## 15 . emp.var.rate 
## 16 . cons.price.idx 
## 17 . cons.conf.idx 
## 18 . euribor3m 
## 19 . nr.employed 
## 
## 
## Variables Entered/Removed: 
## 
## - nr.employed added 
## - month added 
## - poutcome added 
## - emp.var.rate added 
## - contact added 
## - cons.conf.idx added 
## - job added 
## - euribor3m added 
## - day_of_week added 
## - default added 
## - education added 
## - previous added 
## - campaign added 
## - housing added 
## - marital added 
## 
## No more variables to be added or removed.

subsettrain<-c("y","nr.employed","month","poutcome","emp.var.rate","job","contact","cons.conf.idx","euribor3m","day_of_week","default","marital","education","housing","age")
train_lognn <-train_data[subsettrain]

subsettest<-c("y","nr.employed","month","poutcome","emp.var.rate","job","contact","cons.conf.idx","euribor3m","day_of_week","default","marital","education","housing","age")
test_lognn<-test_data[subsettest]


set.seed(888)
logit_model <- glm(y ~.,family=binomial(link='logit'),data =train_lognn)
summary(logit_model)

## 
## Call:
## glm(formula = y ~ ., family = binomial(link = "logit"), data = train_lognn)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -3.0569  -0.9234  -0.5184   0.9164   2.1376  
## 
## Coefficients:
##                                Estimate Std. Error z value Pr(>|z|)    
## (Intercept)                   2.163e+01  1.986e+00  10.890  < 2e-16 ***
## nr.employed                  -3.907e-03  3.907e-04 -10.000  < 2e-16 ***
## monthaug                     -4.492e-01  9.860e-02  -4.555 5.23e-06 ***
## monthdec                      1.140e+00  3.835e-01   2.973 0.002946 ** 
## monthjul                      9.120e-02  8.514e-02   1.071 0.284092    
## monthjun                      1.311e-01  8.248e-02   1.589 0.111993    
## monthmar                      1.317e+00  1.606e-01   8.205 2.30e-16 ***
## monthmay                     -6.927e-01  6.867e-02 -10.087  < 2e-16 ***
## monthnov                     -4.975e-01  8.572e-02  -5.804 6.48e-09 ***
## monthoct                      2.127e+00  2.329e-01   9.133  < 2e-16 ***
## monthsep                     -2.447e-01  1.611e-01  -1.519 0.128657    
## poutcomenonexistent           4.919e-01  6.145e-02   8.005 1.20e-15 ***
## poutcomesuccess               1.922e+00  1.298e-01  14.807  < 2e-16 ***
## emp.var.rate                 -1.283e-01  1.905e-02  -6.739 1.60e-11 ***
## jobblue-collar               -6.148e-02  6.621e-02  -0.928 0.353151    
## jobentrepreneur               7.651e-02  9.990e-02   0.766 0.443781    
## jobhousemaid                  8.607e-02  1.291e-01   0.667 0.505075    
## jobmanagement                -1.029e-01  7.387e-02  -1.393 0.163615    
## jobretired                    2.854e-01  1.062e-01   2.688 0.007193 ** 
## jobself-employed              7.674e-03  9.772e-02   0.079 0.937403    
## jobservices                  -8.800e-02  7.180e-02  -1.226 0.220352    
## jobstudent                    2.705e-01  1.262e-01   2.144 0.032025 *  
## jobtechnician                -5.049e-02  6.067e-02  -0.832 0.405269    
## jobunemployed                -1.063e-01  1.137e-01  -0.935 0.349570    
## contacttelephone             -5.560e-01  6.028e-02  -9.224  < 2e-16 ***
## cons.conf.idx                 3.533e-02  4.825e-03   7.323 2.42e-13 ***
## euribor3m                    -1.299e-01  1.861e-02  -6.980 2.94e-12 ***
## day_of_weekmon               -2.031e-01  5.665e-02  -3.586 0.000336 ***
## day_of_weekthu                3.870e-02  5.592e-02   0.692 0.488849    
## day_of_weektue               -2.467e-02  5.735e-02  -0.430 0.667128    
## day_of_weekwed                5.614e-02  5.642e-02   0.995 0.319641    
## defaultrefuse2disclose       -2.199e-01  5.004e-02  -4.395 1.11e-05 ***
## defaultyes                   -9.872e+00  1.195e+02  -0.083 0.934140    
## maritalmarried                5.248e-02  5.851e-02   0.897 0.369771    
## maritalsingle                 1.246e-01  6.531e-02   1.908 0.056352 .  
## educationbasic.6y             3.173e-01  9.626e-02   3.297 0.000979 ***
## educationbasic.9y             8.142e-03  7.831e-02   0.104 0.917192    
## educationhigh.school          1.619e-01  8.093e-02   2.001 0.045391 *  
## educationilliterate           1.143e+00  7.477e-01   1.528 0.126463    
## educationprofessional.course  1.509e-01  8.880e-02   1.699 0.089256 .  
## educationuniversity.degree    2.113e-01  8.193e-02   2.579 0.009906 ** 
## housingrefuse2disclose       -2.425e-01  1.236e-01  -1.962 0.049779 *  
## housingyes                   -1.056e-01  3.570e-02  -2.958 0.003092 ** 
## age                           2.754e-03  1.922e-03   1.433 0.151986    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 23825  on 17185  degrees of freedom
## Residual deviance: 19310  on 17142  degrees of freedom
## AIC: 19398
## 
## Number of Fisher Scoring iterations: 9

anova(logit_model, test="Chisq")

## Analysis of Deviance Table
## 
## Model: binomial, link: logit
## 
## Response: y
## 
## Terms added sequentially (first to last)
## 
## 
##               Df Deviance Resid. Df Resid. Dev  Pr(>Chi)    
## NULL                          17185      23825              
## nr.employed    1  2708.42     17184      21116 < 2.2e-16 ***
## month          9   930.93     17175      20185 < 2.2e-16 ***
## poutcome       2   313.37     17173      19872 < 2.2e-16 ***
## emp.var.rate   1   263.29     17172      19609 < 2.2e-16 ***
## job           10    36.83     17162      19572 6.067e-05 ***
## contact        1    70.48     17161      19501 < 2.2e-16 ***
## cons.conf.idx  1    46.69     17160      19455 8.301e-12 ***
## euribor3m      1    53.31     17159      19401 2.844e-13 ***
## day_of_week    4    30.44     17155      19371 3.975e-06 ***
## default        2    21.71     17153      19349 1.927e-05 ***
## marital        2     3.80     17151      19345  0.149220    
## education      6    21.98     17145      19323  0.001220 ** 
## housing        2    10.97     17143      19312  0.004148 ** 
## age            1     2.05     17142      19310  0.151910    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

#variable importance
km<-varImp(logit_model)

library(data.table)

## 
## Attaching package: 'data.table'

## The following object is masked from 'package:purrr':
## 
##     transpose

## The following objects are masked from 'package:dplyr':
## 
##     between, first, last

setDT(km, keep.rownames = TRUE)[]

##                               rn     Overall
##  1:                  nr.employed  9.99968850
##  2:                     monthaug  4.55542864
##  3:                     monthdec  2.97335774
##  4:                     monthjul  1.07117311
##  5:                     monthjun  1.58929922
##  6:                     monthmar  8.20521908
##  7:                     monthmay 10.08715780
##  8:                     monthnov  5.80384830
##  9:                     monthoct  9.13266431
## 10:                     monthsep  1.51941852
## 11:          poutcomenonexistent  8.00452506
## 12:              poutcomesuccess 14.80701963
## 13:                 emp.var.rate  6.73857473
## 14:               jobblue-collar  0.92849518
## 15:              jobentrepreneur  0.76582475
## 16:                 jobhousemaid  0.66652650
## 17:                jobmanagement  1.39301448
## 18:                   jobretired  2.68779563
## 19:             jobself-employed  0.07853391
## 20:                  jobservices  1.22559294
## 21:                   jobstudent  2.14410048
## 22:                jobtechnician  0.83224752
## 23:                jobunemployed  0.93542305
## 24:             contacttelephone  9.22433057
## 25:                cons.conf.idx  7.32323678
## 26:                    euribor3m  6.98042679
## 27:               day_of_weekmon  3.58554069
## 28:               day_of_weekthu  0.69214099
## 29:               day_of_weektue  0.43009319
## 30:               day_of_weekwed  0.99519672
## 31:       defaultrefuse2disclose  4.39459091
## 32:                   defaultyes  0.08263706
## 33:               maritalmarried  0.89690227
## 34:                maritalsingle  1.90830303
## 35:            educationbasic.6y  3.29660481
## 36:            educationbasic.9y  0.10397109
## 37:         educationhigh.school  2.00101619
## 38:          educationilliterate  1.52820073
## 39: educationprofessional.course  1.69933324
## 40:   educationuniversity.degree  2.57909182
## 41:       housingrefuse2disclose  1.96185626
## 42:                   housingyes  2.95843720
## 43:                          age  1.43255119
##                               rn     Overall

colnames(km)[2]<-"Importance"
colnames(km)[1]<-"Variables"
km %>%
  arrange(Importance) %>%    # First sort by val. This sort the dataframe but NOT the factor levels
  mutate(name=fct_reorder(Variables, desc(Importance))) %>%   # This trick update the factor levels
  ggplot( aes(x=Variables, y=Importance)) +
  geom_segment( aes(xend=name, yend=0)) +
  geom_point( size=4, color="orange") +
  coord_flip() +
  theme_bw() +
  xlab("")

  #confusion matrix for train
  log.pred.train <-predict(logit_model,data=train_lognn,type="response")
  log.pred1.train <-ifelse(log.pred.train>0.5,1,0)
  log.confusion.matrix.train <-table(log.pred1.train,train_lognn$y)
  log.confusion.matrix.train

##                
## log.pred1.train    0    1
##               0 7008 3249
##               1 1619 5310

  log.accuracy.train=sum(diag(log.confusion.matrix.train))/sum(log.confusion.matrix.train)
  log.accuracy.train

## [1] 0.7167462

  #confusion matrix for test
  log.pred.test <-predict(logit_model,data=test_lognn,type="response")
  log.pred1.test <-ifelse(log.pred.test>0.5,1,0)
  
  error1 <-mean(log.pred1.test !=test_lognn$y)

## Warning in `!=.default`(log.pred1.test, test_lognn$y): longer object length
## is not a multiple of shorter object length

## Warning in is.na(e1) | is.na(e2): longer object length is not a multiple of
## shorter object length

  print(paste('Accuracy',1-error1))

## [1] "Accuracy 0.716571628069359"

###CART
  set.seed(888)
tree_model <- rpart(y ~ ., data = train_data,method="class")
tree_model

## n= 17186 
## 
## node), split, n, loss, yval, (yprob)
##       * denotes terminal node
## 
##  1) root 17186 8559 0 (0.50197835 0.49802165)  
##    2) pdays< 0.08737967 13676 5256 0 (0.61567710 0.38432290)  
##      4) pdays>=-0.08416085 11296 3018 0 (0.73282578 0.26717422)  
##        8) euribor3m>=3.354868 7063 1075 0 (0.84779839 0.15220161) *
##        9) euribor3m< 3.354868 4233 1943 0 (0.54098748 0.45901252)  
##         18) nr.employed>=5054.815 3158 1184 0 (0.62507916 0.37492084) *
##         19) nr.employed< 5054.815 1075  316 1 (0.29395349 0.70604651) *
##      5) pdays< -0.08416085 2380  142 1 (0.05966387 0.94033613) *
##    3) pdays>=0.08737967 3510  207 1 (0.05897436 0.94102564) *

fancyRpartPlot(tree_model)

#predict train
predictions <- predict(tree_model, train_data, type = "class")

#confusion matrix train
tree.confusion.matrix.train <- prop.table(table(predictions, train_data$y))
tree.confusion.matrix.train

##            
## predictions          0          1
##           0 0.46328407 0.13144420
##           1 0.03869429 0.36657745

CrossTable(train_data$y, predictions,
           prop.chisq = FALSE, prop.c = FALSE, prop.r = FALSE,
           dnn = c('actual subscription status', 'predicted subscription status'))

## 
##  
##    Cell Contents
## |-------------------------|
## |                       N |
## |         N / Table Total |
## |-------------------------|
## 
##  
## Total Observations in Table:  17186 
## 
##  
##                            | predicted subscription status 
## actual subscription status |         0 |         1 | Row Total | 
## ---------------------------|-----------|-----------|-----------|
##                          0 |      7962 |       665 |      8627 | 
##                            |     0.463 |     0.039 |           | 
## ---------------------------|-----------|-----------|-----------|
##                          1 |      2259 |      6300 |      8559 | 
##                            |     0.131 |     0.367 |           | 
## ---------------------------|-----------|-----------|-----------|
##               Column Total |     10221 |      6965 |     17186 | 
## ---------------------------|-----------|-----------|-----------|
## 
## 

  #train accuracy
tree.accuracy.train=sum(diag(tree.confusion.matrix.train))/sum(tree.confusion.matrix.train)
tree.accuracy.train

## [1] 0.8298615

#predict test
cart_pred <- predict(tree_model , test_data,type="class")



  # Confusion matrix for test
tree.confusion.matrix.test <- prop.table(table(cart_pred, test_data$y))
tree.confusion.matrix.test

##          
## cart_pred          0          1
##         0 0.46211592 0.13844274
##         1 0.03986266 0.35957868

#test accuracy
tree.accuracy.test=sum(diag(tree.confusion.matrix.test))/sum(tree.confusion.matrix.test)
tree.accuracy.test

## [1] 0.8216946

  # Cross table validation for test
CrossTable(test_data$y, cart_pred,
           prop.chisq = FALSE, prop.c = FALSE, prop.r = FALSE,
           dnn = c('actual customers responses', 'predicted customers responses'))

## 
##  
##    Cell Contents
## |-------------------------|
## |                       N |
## |         N / Table Total |
## |-------------------------|
## 
##  
## Total Observations in Table:  17184 
## 
##  
##                            | predicted customers responses 
## actual customers responses |         0 |         1 | Row Total | 
## ---------------------------|-----------|-----------|-----------|
##                          0 |      7941 |       685 |      8626 | 
##                            |     0.462 |     0.040 |           | 
## ---------------------------|-----------|-----------|-----------|
##                          1 |      2379 |      6179 |      8558 | 
##                            |     0.138 |     0.360 |           | 
## ---------------------------|-----------|-----------|-----------|
##               Column Total |     10320 |      6864 |     17184 | 
## ---------------------------|-----------|-----------|-----------|
## 
## 

##prune tree
set.seed(888)

printcp(tree_model)

## 
## Classification tree:
## rpart(formula = y ~ ., data = train_data, method = "class")
## 
## Variables actually used in tree construction:
## [1] euribor3m   nr.employed pdays      
## 
## Root node error: 8559/17186 = 0.49802
## 
## n= 17186 
## 
##         CP nsplit rel error  xerror      xstd
## 1 0.361725      0   1.00000 1.00000 0.0076583
## 2 0.244888      1   0.63828 0.63792 0.0071312
## 3 0.025879      2   0.39339 0.39514 0.0060895
## 4 0.010000      4   0.34163 0.34245 0.0057608

plotcp(tree_model)

tree_model$cptable[which.min(tree_model$cptable[,"xerror"]),"CP"]

## [1] 0.01

bestcp <- tree_model$cptable[which.min(tree_model$cptable[,"xerror"]),"CP"]
tree.pruned <- prune(tree_model, cp = bestcp)

fancyRpartPlot(tree.pruned)

summary(tree.pruned)

## Call:
## rpart(formula = y ~ ., data = train_data, method = "class")
##   n= 17186 
## 
##           CP nsplit rel error    xerror        xstd
## 1 0.36172450      0 1.0000000 1.0000000 0.007658275
## 2 0.24488842      1 0.6382755 0.6379250 0.007131171
## 3 0.02587919      2 0.3933871 0.3951396 0.006089461
## 4 0.01000000      4 0.3416287 0.3424465 0.005760777
## 
## Variable importance
##          pdays    nr.employed       poutcome      euribor3m   emp.var.rate 
##             54              9              9              8              7 
## cons.price.idx  cons.conf.idx          month       previous 
##              5              4              3              1 
## 
## Node number 1: 17186 observations,    complexity param=0.3617245
##   predicted class=0  expected loss=0.4980216  P(node) =1
##     class counts:  8627  8559
##    probabilities: 0.502 0.498 
##   left son=2 (13676 obs) right son=3 (3510 obs)
##   Primary splits:
##       pdays        < 0.08737967  to the left,  improve=1731.2830, (0 missing)
##       euribor3m    < 3.301685    to the right, improve=1362.9070, (0 missing)
##       nr.employed  < 5159.954    to the right, improve=1257.1140, (0 missing)
##       emp.var.rate < -0.4153285  to the right, improve=1186.8380, (0 missing)
##       month        splits as  RLRLLRLLRR, improve= 655.5931, (0 missing)
##   Surrogate splits:
##       poutcome      splits as  LLR, agree=0.862, adj=0.326, (0 split)
##       previous      < 1.160638    to the left,  agree=0.801, adj=0.027, (0 split)
##       emp.var.rate  < -4.180714   to the right, agree=0.797, adj=0.004, (0 split)
##       nr.employed   < 4904.317    to the right, agree=0.796, adj=0.002, (0 split)
##       cons.conf.idx < -27.60867   to the left,  agree=0.796, adj=0.002, (0 split)
## 
## Node number 2: 13676 observations,    complexity param=0.2448884
##   predicted class=0  expected loss=0.3843229  P(node) =0.795764
##     class counts:  8420  5256
##    probabilities: 0.616 0.384 
##   left son=4 (11296 obs) right son=5 (2380 obs)
##   Primary splits:
##       pdays        < -0.08416085 to the right, improve=1781.6060, (0 missing)
##       euribor3m    < 3.243719    to the right, improve= 814.4408, (0 missing)
##       nr.employed  < 5151.223    to the right, improve= 747.7505, (0 missing)
##       emp.var.rate < -0.4251875  to the right, improve= 725.9074, (0 missing)
##       month        splits as  RLRLLRLLRR, improve= 476.8494, (0 missing)
##   Surrogate splits:
##       cons.price.idx < 95.14196    to the left,  agree=0.826, adj=0.003, (0 split)
##       euribor3m      < 7.180609    to the left,  agree=0.826, adj=0.002, (0 split)
##       previous       < 1.416851    to the left,  agree=0.826, adj=0.001, (0 split)
##       nr.employed    < 5338.901    to the left,  agree=0.826, adj=0.001, (0 split)
##       age            < 9.482672    to the right, agree=0.826, adj=0.001, (0 split)
## 
## Node number 3: 3510 observations
##   predicted class=1  expected loss=0.05897436  P(node) =0.204236
##     class counts:   207  3303
##    probabilities: 0.059 0.941 
## 
## Node number 4: 11296 observations,    complexity param=0.02587919
##   predicted class=0  expected loss=0.2671742  P(node) =0.6572792
##     class counts:  8278  3018
##    probabilities: 0.733 0.267 
##   left son=8 (7063 obs) right son=9 (4233 obs)
##   Primary splits:
##       euribor3m    < 3.354868    to the right, improve=498.2925, (0 missing)
##       nr.employed  < 5063.45     to the right, improve=481.6256, (0 missing)
##       emp.var.rate < -0.4153285  to the right, improve=444.1400, (0 missing)
##       month        splits as  RLRLLRLLRR, improve=343.2295, (0 missing)
##       contact      splits as  RL, improve=167.5603, (0 missing)
##   Surrogate splits:
##       nr.employed    < 5147.75     to the right, agree=0.950, adj=0.867, (0 split)
##       emp.var.rate   < -0.6097237  to the right, agree=0.948, adj=0.861, (0 split)
##       cons.price.idx < 93.18318    to the right, agree=0.832, adj=0.551, (0 split)
##       cons.conf.idx  < -44.74206   to the right, agree=0.790, adj=0.440, (0 split)
##       month          splits as  RLRLLRLLRR, agree=0.746, adj=0.321, (0 split)
## 
## Node number 5: 2380 observations
##   predicted class=1  expected loss=0.05966387  P(node) =0.1384848
##     class counts:   142  2238
##    probabilities: 0.060 0.940 
## 
## Node number 8: 7063 observations
##   predicted class=0  expected loss=0.1522016  P(node) =0.410974
##     class counts:  5988  1075
##    probabilities: 0.848 0.152 
## 
## Node number 9: 4233 observations,    complexity param=0.02587919
##   predicted class=0  expected loss=0.4590125  P(node) =0.2463051
##     class counts:  2290  1943
##    probabilities: 0.541 0.459 
##   left son=18 (3158 obs) right son=19 (1075 obs)
##   Primary splits:
##       nr.employed    < 5054.815    to the right, improve=175.86850, (0 missing)
##       month          splits as  LRRRRRLLRR, improve=172.38340, (0 missing)
##       cons.conf.idx  < -42.51436   to the left,  improve=130.53500, (0 missing)
##       cons.price.idx < 93.34726    to the left,  improve= 82.87013, (0 missing)
##       job            splits as  RLLRLRRLRRR, improve= 69.75490, (0 missing)
##   Surrogate splits:
##       cons.price.idx < 93.58645    to the left,  agree=0.822, adj=0.299, (0 split)
##       month          splits as  LLRRLLLLRR, agree=0.802, adj=0.219, (0 split)
##       cons.conf.idx  < -39.36025   to the left,  agree=0.790, adj=0.175, (0 split)
##       euribor3m      < -0.06928759 to the right, agree=0.756, adj=0.040, (0 split)
##       age            < 63.35893    to the left,  agree=0.748, adj=0.009, (0 split)
## 
## Node number 18: 3158 observations
##   predicted class=0  expected loss=0.3749208  P(node) =0.1837542
##     class counts:  1974  1184
##    probabilities: 0.625 0.375 
## 
## Node number 19: 1075 observations
##   predicted class=1  expected loss=0.2939535  P(node) =0.06255091
##     class counts:   316   759
##    probabilities: 0.294 0.706

#variable importance
argPlot <- as.data.frame(tree.pruned$variable.importance)
df <- data.frame(imp = tree.pruned$variable.importance)
df2 <- df %>% 
  tibble::rownames_to_column() %>% 
  dplyr::rename("variable" = rowname) %>% 
  dplyr::arrange(imp) %>%
  dplyr::mutate(variable = forcats::fct_inorder(variable))

ggplot2::ggplot(df2) +
  geom_segment(aes(x = variable, y = 0, xend = variable, yend = imp), 
               size = 1.5, alpha = 0.7) +
  geom_point(aes(x = variable, y = imp, col = variable), 
             size = 4, show.legend = F)  +
  coord_flip() + labs(x="Variables",y="Importance")+
  theme_bw()

# Compute the train accuracy of the pruned tree

train_data$pred <- predict(tree.pruned, train_data, type = "class")
accuracy_prun_train <- mean(train_data$pred == train_data$y)
accuracy_prun_train

## [1] 0.8298615

pruned.confusion.matrix.train <- prop.table(table(train_data$pred, train_data$y))
pruned.confusion.matrix.train

##    
##              0          1
##   0 0.46328407 0.13144420
##   1 0.03869429 0.36657745

# Compute the test accuracy of the pruned tree
test_data$pred <- predict(tree.pruned, test_data, type = "class")
accuracy_prune_test <- mean(test_data$pred== test_data$y)
accuracy_prune_test

## [1] 0.8216946

pruned.confusion.matrix.test <- prop.table(table(test_data$pred, test_data$y))
pruned.confusion.matrix.test

##    
##              0          1
##   0 0.46211592 0.13844274
##   1 0.03986266 0.35957868

#The tree after being pruned is the same as before


#=============================================================

#Random Forest
library(randomForest)

## randomForest 4.6-14

## Type rfNews() to see new features/changes/bug fixes.

## 
## Attaching package: 'randomForest'

## The following object is masked from 'package:rattle':
## 
##     importance

## The following object is masked from 'package:gridExtra':
## 
##     combine

## The following object is masked from 'package:ggplot2':
## 
##     margin

## The following object is masked from 'package:dplyr':
## 
##     combine

set.seed(888)
RF.model <- randomForest(y~., data=train_data, ntree=100, importance=TRUE)
RF.model

## 
## Call:
##  randomForest(formula = y ~ ., data = train_data, ntree = 100,      importance = TRUE) 
##                Type of random forest: classification
##                      Number of trees: 100
## No. of variables tried at each split: 4
## 
##         OOB estimate of  error rate: 13.03%
## Confusion matrix:
##      0    1 class.error
## 0 7667  960   0.1112785
## 1 1279 7280   0.1494333

summary(RF.model)

##                 Length Class  Mode     
## call                5  -none- call     
## type                1  -none- character
## predicted       17186  factor numeric  
## err.rate          300  -none- numeric  
## confusion           6  -none- numeric  
## votes           34372  matrix numeric  
## oob.times       17186  -none- numeric  
## classes             2  -none- character
## importance         80  -none- numeric  
## importanceSD       60  -none- numeric  
## localImportance     0  -none- NULL     
## proximity           0  -none- NULL     
## ntree               1  -none- numeric  
## mtry                1  -none- numeric  
## forest             14  -none- list     
## y               17186  factor numeric  
## test                0  -none- NULL     
## inbag               0  -none- NULL     
## terms               3  terms  call

#variable importance
varImpPlot(RF.model)

#Next we display an error plot of the random forest model:
plot(RF.model)

#train
RF.predict.train <- predict(RF.model, newdata = train_data)
RF.train.cm <- as.matrix(table(Actual1 = train_data$y, Predicted1 = RF.predict.train))
RF.train.cm

##        Predicted1
## Actual1    0    1
##       0 8627    0
##       1    0 8559

accuracy_train_rf=sum(diag(RF.train.cm))/sum(RF.train.cm)
accuracy_train_rf

## [1] 1

#test
RF.predict <- predict(RF.model, newdata = test_data)
RF.cm <- as.matrix(table(Actual = test_data$y, Predicted = RF.predict))
RF.cm

##       Predicted
## Actual    0    1
##      0 7686  940
##      1 1343 7215

accuracy_test_rf=sum(diag(RF.cm))/sum(RF.cm)
accuracy_test_rf

## [1] 0.8671439

#Below we test the accuracy on the training and test datasets and we see that it is 90.87% and 83.45%, respectively. 
#The “out of sample” error is 16.51% and is in agreement with the OOB error:

#library(randomForestExplainer)
#explain_forest(RF.model, interactions = TRUE, data = test_data)



#=============================================


#==============================================================

#neural nets
library(nnet)
library(NeuralNetTools)
library(neuralnet)

## 
## Attaching package: 'neuralnet'

## The following object is masked from 'package:ROCR':
## 
##     prediction

## The following object is masked from 'package:dplyr':
## 
##     compute

## For neural nets, use -- variables(Feature selection step)

#I don't inclue "y" variable, 
              #because I dont want to make it split into 2 column y0 and y1 in 
              #in one-hot-encoding
subsetnn <- subset(balanced_data, select=c("nr.employed","month","poutcome","emp.var.rate","job",
                                           "contact","cons.conf.idx","euribor3m","day_of_week",
                                           "default","marital","education","housing"))

subset_y <- balanced_data$y

#Use of one-hot-encoding to transfer categorical variables into numerical variables
#Since I want to do data scaling before applying nn, 
#and it requires numerical variables
dmy <- dummyVars(" ~ .", data = subsetnn)
bank.dummies<- data.frame(predict(dmy, newdata = subsetnn))

## Scale data for neural network
max = apply(bank.dummies , 2 , max)
min = apply(bank.dummies, 2 , min)
scaled = as.data.frame(scale(bank.dummies, center = min, scale = max - min))

#add back the y variable back to bank.dummies
scaled$y <- as.factor(subset_y)

##data_split

idx <- sample(1:dim(scaled)[1], dim(scaled)[1]/2)
trainNN = scaled[idx , ]
testNN = scaled[-idx , ]

#model fit
set.seed(888)
nn <- train(y ~ .,
                  data = trainNN,
                  method = "nnet")

## # weights:  54
## initial  value 12590.380546 
## iter  10 value 11264.857943
## iter  20 value 11041.077597
## iter  30 value 10359.862778
## iter  40 value 9850.000630
## iter  50 value 9721.984879
## iter  60 value 9687.141551
## iter  70 value 9664.154578
## iter  80 value 9656.488049
## iter  90 value 9639.242090
## iter 100 value 9632.834174
## final  value 9632.834174 
## stopped after 100 iterations
## # weights:  160
## initial  value 14073.263671 
## iter  10 value 11740.846972
## iter  20 value 10885.132441
## iter  30 value 10273.067836
## iter  40 value 9855.307796
## iter  50 value 9622.578266
## iter  60 value 9554.058708
## iter  70 value 9512.528619
## iter  80 value 9485.976982
## iter  90 value 9458.580865
## iter 100 value 9440.308004
## final  value 9440.308004 
## stopped after 100 iterations
## # weights:  266
## initial  value 12659.678671 
## iter  10 value 10143.678562
## iter  20 value 9593.716770
## iter  30 value 9356.987274
## iter  40 value 9242.785893
## iter  50 value 9158.553435
## iter  60 value 8979.444685
## iter  70 value 8875.674758
## iter  80 value 8813.938814
## iter  90 value 8795.827353
## iter 100 value 8780.929956
## final  value 8780.929956 
## stopped after 100 iterations
## # weights:  54
## initial  value 13217.006601 
## iter  10 value 11364.283946
## iter  20 value 10665.854091
## iter  30 value 10236.836466
## iter  40 value 9844.985366
## iter  50 value 9723.062115
## iter  60 value 9633.946478
## iter  70 value 9609.048814
## iter  80 value 9606.438368
## iter  90 value 9603.811271
## iter 100 value 9602.589001
## final  value 9602.589001 
## stopped after 100 iterations
## # weights:  160
## initial  value 12571.296162 
## iter  10 value 10308.252732
## iter  20 value 9607.555319
## iter  30 value 9485.259317
## iter  40 value 9402.226779
## iter  50 value 9362.502165
## iter  60 value 9324.247164
## iter  70 value 9288.203249
## iter  80 value 9244.708854
## iter  90 value 9197.215892
## iter 100 value 9167.596673
## final  value 9167.596673 
## stopped after 100 iterations
## # weights:  266
## initial  value 12506.167296 
## iter  10 value 9868.061295
## iter  20 value 9558.152157
## iter  30 value 9385.529124
## iter  40 value 9207.001383
## iter  50 value 9028.406111
## iter  60 value 8949.502669
## iter  70 value 8907.814721
## iter  80 value 8881.115086
## iter  90 value 8859.900935
## iter 100 value 8851.333062
## final  value 8851.333062 
## stopped after 100 iterations
## # weights:  54
## initial  value 12146.127603 
## iter  10 value 10964.552870
## iter  20 value 10003.248276
## iter  30 value 9737.521076
## iter  40 value 9679.484983
## iter  50 value 9654.619079
## iter  60 value 9651.885181
## iter  70 value 9649.088527
## iter  80 value 9628.794814
## iter  90 value 9595.852334
## iter 100 value 9593.655716
## final  value 9593.655716 
## stopped after 100 iterations
## # weights:  160
## initial  value 12149.623819 
## iter  10 value 10020.240126
## iter  20 value 9560.859036
## iter  30 value 9422.885860
## iter  40 value 9311.549959
## iter  50 value 9159.867166
## iter  60 value 9099.287568
## iter  70 value 9019.425480
## iter  80 value 8980.420634
## iter  90 value 8957.288421
## iter 100 value 8932.545740
## final  value 8932.545740 
## stopped after 100 iterations
## # weights:  266
## initial  value 15970.396828 
## iter  10 value 9890.786306
## iter  20 value 9469.353687
## iter  30 value 9221.075246
## iter  40 value 9005.781520
## iter  50 value 8901.831384
## iter  60 value 8841.329316
## iter  70 value 8798.358554
## iter  80 value 8741.468566
## iter  90 value 8702.617383
## iter 100 value 8680.887254
## final  value 8680.887254 
## stopped after 100 iterations
## # weights:  54
## initial  value 12934.900682 
## iter  10 value 11040.250904
## iter  20 value 10106.077271
## iter  30 value 9975.097015
## iter  40 value 9924.416560
## iter  50 value 9914.821730
## iter  60 value 9909.908826
## iter  70 value 9909.858530
## final  value 9909.854466 
## converged
## # weights:  160
## initial  value 12221.020990 
## iter  10 value 10159.917390
## iter  20 value 9810.553124
## iter  30 value 9545.370396
## iter  40 value 9445.721143
## iter  50 value 9361.486312
## iter  60 value 9327.007007
## iter  70 value 9298.661911
## iter  80 value 9279.207999
## iter  90 value 9256.534284
## iter 100 value 9231.607150
## final  value 9231.607150 
## stopped after 100 iterations
## # weights:  266
## initial  value 12177.758501 
## iter  10 value 10210.573947
## iter  20 value 9790.500876
## iter  30 value 9603.988559
## iter  40 value 9446.636816
## iter  50 value 9283.847067
## iter  60 value 9164.965353
## iter  70 value 9080.046226
## iter  80 value 9011.295954
## iter  90 value 8972.626776
## iter 100 value 8939.157359
## final  value 8939.157359 
## stopped after 100 iterations
## # weights:  54
## initial  value 13015.315563 
## iter  10 value 11129.168701
## iter  20 value 10330.016995
## iter  30 value 9959.029911
## iter  40 value 9850.334419
## iter  50 value 9818.962633
## iter  60 value 9744.415624
## iter  70 value 9707.952034
## iter  80 value 9680.302879
## iter  90 value 9671.227852
## iter 100 value 9665.972410
## final  value 9665.972410 
## stopped after 100 iterations
## # weights:  160
## initial  value 15475.011868 
## iter  10 value 10832.404134
## iter  20 value 9860.678605
## iter  30 value 9647.161253
## iter  40 value 9530.093918
## iter  50 value 9465.810354
## iter  60 value 9420.068109
## iter  70 value 9379.012371
## iter  80 value 9340.445830
## iter  90 value 9322.726042
## iter 100 value 9312.759270
## final  value 9312.759270 
## stopped after 100 iterations
## # weights:  266
## initial  value 12046.790105 
## iter  10 value 9760.175445
## iter  20 value 9500.819079
## iter  30 value 9390.258264
## iter  40 value 9225.066645
## iter  50 value 9148.888509
## iter  60 value 9094.594688
## iter  70 value 8996.909195
## iter  80 value 8895.818408
## iter  90 value 8826.746904
## iter 100 value 8797.141034
## final  value 8797.141034 
## stopped after 100 iterations
## # weights:  54
## initial  value 11985.153117 
## iter  10 value 10335.343086
## iter  20 value 10056.146596
## iter  30 value 9915.751039
## iter  40 value 9865.426977
## iter  50 value 9802.630298
## iter  60 value 9725.145271
## iter  70 value 9689.622188
## iter  80 value 9674.645820
## iter  90 value 9666.795750
## iter 100 value 9658.976334
## final  value 9658.976334 
## stopped after 100 iterations
## # weights:  160
## initial  value 11863.080634 
## iter  10 value 9899.957351
## iter  20 value 9712.134360
## iter  30 value 9614.072702
## iter  40 value 9534.289214
## iter  50 value 9478.638600
## iter  60 value 9439.118346
## iter  70 value 9349.015435
## iter  80 value 9272.594718
## iter  90 value 9225.301950
## iter 100 value 9193.342947
## final  value 9193.342947 
## stopped after 100 iterations
## # weights:  266
## initial  value 12024.752522 
## iter  10 value 9778.826596
## iter  20 value 9603.739403
## iter  30 value 9404.470765
## iter  40 value 9276.797657
## iter  50 value 9220.410439
## iter  60 value 9168.080895
## iter  70 value 9104.775051
## iter  80 value 9060.533056
## iter  90 value 9021.026449
## iter 100 value 8990.356866
## final  value 8990.356866 
## stopped after 100 iterations
## # weights:  54
## initial  value 11921.885806 
## iter  10 value 10695.205349
## iter  20 value 9875.824522
## iter  30 value 9677.190467
## iter  40 value 9622.766449
## iter  50 value 9614.461292
## iter  60 value 9611.977835
## iter  70 value 9611.964015
## final  value 9611.962360 
## converged
## # weights:  160
## initial  value 12150.209663 
## iter  10 value 10454.976822
## iter  20 value 9892.688926
## iter  30 value 9662.069717
## iter  40 value 9578.799661
## iter  50 value 9492.130802
## iter  60 value 9429.246736
## iter  70 value 9372.473618
## iter  80 value 9320.693064
## iter  90 value 9289.179994
## iter 100 value 9242.702082
## final  value 9242.702082 
## stopped after 100 iterations
## # weights:  266
## initial  value 11778.406584 
## iter  10 value 9839.808612
## iter  20 value 9450.998432
## iter  30 value 9229.351500
## iter  40 value 9126.968098
## iter  50 value 9027.169079
## iter  60 value 8946.847041
## iter  70 value 8901.219853
## iter  80 value 8837.813295
## iter  90 value 8785.309451
## iter 100 value 8725.226777
## final  value 8725.226777 
## stopped after 100 iterations
## # weights:  54
## initial  value 11871.027323 
## iter  10 value 10952.469278
## iter  20 value 10674.326246
## iter  30 value 10209.935719
## iter  40 value 9929.395186
## iter  50 value 9725.206998
## iter  60 value 9666.496292
## iter  70 value 9651.594210
## iter  80 value 9627.649037
## iter  90 value 9626.098585
## iter 100 value 9626.019587
## final  value 9626.019587 
## stopped after 100 iterations
## # weights:  160
## initial  value 11954.611151 
## iter  10 value 9947.460419
## iter  20 value 9672.944119
## iter  30 value 9560.781249
## iter  40 value 9430.903228
## iter  50 value 9378.728911
## iter  60 value 9357.668663
## iter  70 value 9344.345697
## iter  80 value 9337.653420
## iter  90 value 9327.641199
## iter 100 value 9315.711873
## final  value 9315.711873 
## stopped after 100 iterations
## # weights:  266
## initial  value 12894.254138 
## iter  10 value 9813.512151
## iter  20 value 9432.526590
## iter  30 value 9311.292807
## iter  40 value 9245.959156
## iter  50 value 9138.085754
## iter  60 value 9078.075677
## iter  70 value 9005.479534
## iter  80 value 8948.630257
## iter  90 value 8887.190460
## iter 100 value 8829.121233
## final  value 8829.121233 
## stopped after 100 iterations
## # weights:  54
## initial  value 11902.002398 
## iter  10 value 10661.487249
## iter  20 value 9907.308364
## iter  30 value 9753.539399
## iter  40 value 9681.866451
## iter  50 value 9665.569066
## iter  60 value 9665.449492
## final  value 9665.433661 
## converged
## # weights:  160
## initial  value 14244.330666 
## iter  10 value 11057.337817
## iter  20 value 9594.681360
## iter  30 value 9428.243630
## iter  40 value 9311.933072
## iter  50 value 9268.843270
## iter  60 value 9182.125911
## iter  70 value 9080.237291
## iter  80 value 9037.614763
## iter  90 value 9009.617881
## iter 100 value 8964.236851
## final  value 8964.236851 
## stopped after 100 iterations
## # weights:  266
## initial  value 12013.980985 
## iter  10 value 9854.827454
## iter  20 value 9437.342268
## iter  30 value 9171.649809
## iter  40 value 8960.957982
## iter  50 value 8742.063018
## iter  60 value 8609.362770
## iter  70 value 8570.690035
## iter  80 value 8543.458705
## iter  90 value 8524.723083
## iter 100 value 8505.907127
## final  value 8505.907127 
## stopped after 100 iterations
## # weights:  54
## initial  value 12062.161751 
## iter  10 value 11627.326069
## iter  20 value 11043.121483
## iter  30 value 10887.696637
## iter  40 value 10836.011991
## iter  50 value 10828.726005
## iter  60 value 10810.729860
## iter  70 value 10598.954198
## iter  80 value 10573.074650
## iter  90 value 10567.878183
## iter 100 value 10544.645733
## final  value 10544.645733 
## stopped after 100 iterations
## # weights:  160
## initial  value 11793.745702 
## iter  10 value 9880.523396
## iter  20 value 9628.733374
## iter  30 value 9451.188695
## iter  40 value 9341.225368
## iter  50 value 9248.585358
## iter  60 value 9182.150153
## iter  70 value 9159.636761
## iter  80 value 9141.606428
## iter  90 value 9083.833461
## iter 100 value 9022.292442
## final  value 9022.292442 
## stopped after 100 iterations
## # weights:  266
## initial  value 12010.084032 
## iter  10 value 9901.414799
## iter  20 value 9549.050392
## iter  30 value 9267.503759
## iter  40 value 9100.624702
## iter  50 value 8977.473187
## iter  60 value 8906.579339
## iter  70 value 8841.563184
## iter  80 value 8765.994825
## iter  90 value 8744.086189
## iter 100 value 8729.897099
## final  value 8729.897099 
## stopped after 100 iterations
## # weights:  54
## initial  value 13437.775766 
## iter  10 value 11914.969542
## iter  20 value 11010.299264
## iter  30 value 10162.900069
## iter  40 value 9956.877041
## iter  50 value 9889.028925
## iter  60 value 9817.042028
## iter  70 value 9684.426867
## iter  80 value 9637.904009
## iter  90 value 9625.589837
## iter 100 value 9613.102144
## final  value 9613.102144 
## stopped after 100 iterations
## # weights:  160
## initial  value 12246.893646 
## iter  10 value 9934.924077
## iter  20 value 9651.496412
## iter  30 value 9468.274924
## iter  40 value 9371.779577
## iter  50 value 9334.466359
## iter  60 value 9286.635379
## iter  70 value 9220.401758
## iter  80 value 9187.546313
## iter  90 value 9112.989861
## iter 100 value 9036.453010
## final  value 9036.453010 
## stopped after 100 iterations
## # weights:  266
## initial  value 11800.218892 
## iter  10 value 9762.785083
## iter  20 value 9518.072327
## iter  30 value 9363.440622
## iter  40 value 9258.606124
## iter  50 value 9133.150638
## iter  60 value 9007.795389
## iter  70 value 8843.019600
## iter  80 value 8769.846152
## iter  90 value 8743.038824
## iter 100 value 8725.363515
## final  value 8725.363515 
## stopped after 100 iterations
## # weights:  54
## initial  value 12242.913873 
## iter  10 value 10364.224244
## iter  20 value 10021.479125
## iter  30 value 9947.202871
## iter  40 value 9912.313491
## iter  50 value 9902.252242
## iter  60 value 9901.813225
## iter  70 value 9901.780233
## iter  80 value 9901.553126
## iter  90 value 9763.412774
## iter 100 value 9674.276146
## final  value 9674.276146 
## stopped after 100 iterations
## # weights:  160
## initial  value 12748.997894 
## iter  10 value 10841.204851
## iter  20 value 10260.469172
## iter  30 value 9760.918543
## iter  40 value 9689.167697
## iter  50 value 9664.006581
## iter  60 value 9657.502233
## iter  70 value 9654.719031
## iter  80 value 9649.665896
## iter  90 value 9632.324981
## iter 100 value 9629.902361
## final  value 9629.902361 
## stopped after 100 iterations
## # weights:  266
## initial  value 12755.523220 
## iter  10 value 9715.079011
## iter  20 value 9434.172080
## iter  30 value 9266.813228
## iter  40 value 9149.709467
## iter  50 value 8983.386098
## iter  60 value 8858.170352
## iter  70 value 8775.004825
## iter  80 value 8722.533953
## iter  90 value 8676.798970
## iter 100 value 8631.850468
## final  value 8631.850468 
## stopped after 100 iterations
## # weights:  54
## initial  value 11917.929419 
## iter  10 value 10505.809361
## iter  20 value 9872.428860
## iter  30 value 9753.890708
## iter  40 value 9724.498084
## iter  50 value 9716.113790
## iter  60 value 9711.942040
## iter  70 value 9706.938193
## iter  80 value 9692.283213
## iter  90 value 9674.663888
## iter 100 value 9664.970253
## final  value 9664.970253 
## stopped after 100 iterations
## # weights:  160
## initial  value 14143.151222 
## iter  10 value 10138.362626
## iter  20 value 9610.282776
## iter  30 value 9369.117725
## iter  40 value 9200.950104
## iter  50 value 9074.509834
## iter  60 value 9001.116612
## iter  70 value 8908.528354
## iter  80 value 8842.233647
## iter  90 value 8821.859803
## iter 100 value 8813.144657
## final  value 8813.144657 
## stopped after 100 iterations
## # weights:  266
## initial  value 12095.526160 
## iter  10 value 9995.430812
## iter  20 value 9565.209945
## iter  30 value 9416.404846
## iter  40 value 9324.688666
## iter  50 value 9162.318808
## iter  60 value 8952.811078
## iter  70 value 8708.345233
## iter  80 value 8584.830564
## iter  90 value 8536.739584
## iter 100 value 8489.029707
## final  value 8489.029707 
## stopped after 100 iterations
## # weights:  54
## initial  value 12211.751162 
## iter  10 value 10522.910980
## iter  20 value 9883.391448
## iter  30 value 9744.509150
## iter  40 value 9684.021166
## iter  50 value 9644.059441
## iter  60 value 9616.892182
## iter  70 value 9597.055490
## iter  80 value 9586.736552
## iter  90 value 9586.396532
## final  value 9586.390413 
## converged
## # weights:  160
## initial  value 12956.508392 
## iter  10 value 10483.313191
## iter  20 value 9920.443216
## iter  30 value 9692.592295
## iter  40 value 9630.540950
## iter  50 value 9556.425435
## iter  60 value 9502.991728
## iter  70 value 9467.693419
## iter  80 value 9455.549788
## iter  90 value 9442.909070
## iter 100 value 9412.745249
## final  value 9412.745249 
## stopped after 100 iterations
## # weights:  266
## initial  value 14742.151332 
## iter  10 value 9821.917196
## iter  20 value 9428.314974
## iter  30 value 9264.464593
## iter  40 value 9148.610248
## iter  50 value 9077.953754
## iter  60 value 9027.397095
## iter  70 value 9001.512154
## iter  80 value 8982.823920
## iter  90 value 8960.081748
## iter 100 value 8908.170673
## final  value 8908.170673 
## stopped after 100 iterations
## # weights:  54
## initial  value 12190.808100 
## iter  10 value 11109.882229
## iter  20 value 10035.863863
## iter  30 value 9748.069486
## iter  40 value 9690.144733
## iter  50 value 9637.126708
## iter  60 value 9619.676342
## iter  70 value 9599.302863
## iter  80 value 9573.470609
## iter  90 value 9573.407944
## iter 100 value 9573.379636
## final  value 9573.379636 
## stopped after 100 iterations
## # weights:  160
## initial  value 12078.468390 
## iter  10 value 10058.406198
## iter  20 value 9634.475589
## iter  30 value 9475.017191
## iter  40 value 9362.865243
## iter  50 value 9276.947493
## iter  60 value 9214.261123
## iter  70 value 9185.173091
## iter  80 value 9169.096225
## iter  90 value 9126.179916
## iter 100 value 9091.796622
## final  value 9091.796622 
## stopped after 100 iterations
## # weights:  266
## initial  value 12286.906271 
## iter  10 value 10139.650111
## iter  20 value 9622.540252
## iter  30 value 9335.815556
## iter  40 value 9204.916712
## iter  50 value 9158.669644
## iter  60 value 9081.313741
## iter  70 value 8967.790173
## iter  80 value 8902.374281
## iter  90 value 8820.228973
## iter 100 value 8757.676229
## final  value 8757.676229 
## stopped after 100 iterations
## # weights:  54
## initial  value 11991.069531 
## iter  10 value 10059.031091
## iter  20 value 9713.370400
## iter  30 value 9631.045151
## iter  40 value 9605.999658
## iter  50 value 9574.207409
## iter  60 value 9550.453731
## iter  70 value 9549.695046
## iter  80 value 9549.583649
## iter  90 value 9548.798117
## iter 100 value 9546.370710
## final  value 9546.370710 
## stopped after 100 iterations
## # weights:  160
## initial  value 13574.537433 
## iter  10 value 11200.448971
## iter  20 value 10094.225729
## iter  30 value 9787.028664
## iter  40 value 9594.252283
## iter  50 value 9527.399586
## iter  60 value 9496.982009
## iter  70 value 9489.922368
## iter  80 value 9489.301885
## iter  90 value 9489.286722
## iter 100 value 9489.271040
## final  value 9489.271040 
## stopped after 100 iterations
## # weights:  266
## initial  value 13915.852657 
## iter  10 value 9615.389121
## iter  20 value 9314.593323
## iter  30 value 9183.656398
## iter  40 value 9077.767985
## iter  50 value 8940.565283
## iter  60 value 8869.647866
## iter  70 value 8797.572512
## iter  80 value 8683.439362
## iter  90 value 8643.275336
## iter 100 value 8616.760647
## final  value 8616.760647 
## stopped after 100 iterations
## # weights:  54
## initial  value 12079.403364 
## iter  10 value 9860.021576
## iter  20 value 9806.294263
## iter  30 value 9775.527475
## iter  40 value 9769.738741
## iter  50 value 9764.389890
## iter  60 value 9661.561968
## iter  70 value 9597.030263
## iter  80 value 9527.120617
## iter  90 value 9503.877640
## iter 100 value 9500.647596
## final  value 9500.647596 
## stopped after 100 iterations
## # weights:  160
## initial  value 14436.139071 
## iter  10 value 10067.789366
## iter  20 value 9775.373665
## iter  30 value 9581.820661
## iter  40 value 9514.074052
## iter  50 value 9413.999088
## iter  60 value 9335.989754
## iter  70 value 9241.288851
## iter  80 value 9185.293743
## iter  90 value 9149.994245
## iter 100 value 9130.013491
## final  value 9130.013491 
## stopped after 100 iterations
## # weights:  266
## initial  value 11632.383198 
## iter  10 value 9792.016815
## iter  20 value 9492.130106
## iter  30 value 9352.859577
## iter  40 value 9281.840826
## iter  50 value 9178.846073
## iter  60 value 9077.307675
## iter  70 value 9004.454476
## iter  80 value 8897.695331
## iter  90 value 8769.490045
## iter 100 value 8696.383487
## final  value 8696.383487 
## stopped after 100 iterations
## # weights:  54
## initial  value 11839.088279 
## iter  10 value 10130.355155
## iter  20 value 9809.558782
## iter  30 value 9777.469977
## iter  40 value 9763.081836
## iter  50 value 9753.244229
## iter  60 value 9752.562899
## final  value 9752.543852 
## converged
## # weights:  160
## initial  value 12160.762781 
## iter  10 value 9786.552446
## iter  20 value 9627.193499
## iter  30 value 9497.172086
## iter  40 value 9408.476710
## iter  50 value 9333.980961
## iter  60 value 9256.248382
## iter  70 value 9162.470810
## iter  80 value 9071.250785
## iter  90 value 9043.648564
## iter 100 value 9006.059974
## final  value 9006.059974 
## stopped after 100 iterations
## # weights:  266
## initial  value 11886.737337 
## iter  10 value 9702.322432
## iter  20 value 9425.035204
## iter  30 value 9260.550692
## iter  40 value 9121.218720
## iter  50 value 8997.384765
## iter  60 value 8825.490862
## iter  70 value 8687.208674
## iter  80 value 8630.130447
## iter  90 value 8593.787506
## iter 100 value 8564.359309
## final  value 8564.359309 
## stopped after 100 iterations
## # weights:  54
## initial  value 11827.345988 
## iter  10 value 9995.829885
## iter  20 value 9699.230213
## iter  30 value 9628.873639
## iter  40 value 9618.593521
## iter  50 value 9614.172476
## iter  60 value 9614.074955
## iter  70 value 9614.041802
## iter  80 value 9612.128394
## iter  90 value 9609.051022
## iter 100 value 9607.370468
## final  value 9607.370468 
## stopped after 100 iterations
## # weights:  160
## initial  value 12874.455390 
## iter  10 value 9881.885003
## iter  20 value 9684.113159
## iter  30 value 9513.934064
## iter  40 value 9428.468196
## iter  50 value 9350.283224
## iter  60 value 9266.276375
## iter  70 value 9231.891642
## iter  80 value 9219.034531
## iter  90 value 9211.818498
## iter 100 value 9188.312920
## final  value 9188.312920 
## stopped after 100 iterations
## # weights:  266
## initial  value 11922.585564 
## iter  10 value 9991.529335
## iter  20 value 9457.859339
## iter  30 value 9178.543472
## iter  40 value 9039.654840
## iter  50 value 8985.411112
## iter  60 value 8935.097588
## iter  70 value 8888.194688
## iter  80 value 8834.817838
## iter  90 value 8779.267634
## iter 100 value 8706.302389
## final  value 8706.302389 
## stopped after 100 iterations
## # weights:  54
## initial  value 12554.423128 
## iter  10 value 10802.710250
## iter  20 value 10435.383073
## iter  30 value 9808.688831
## iter  40 value 9631.199506
## iter  50 value 9586.001518
## iter  60 value 9573.256021
## iter  70 value 9570.699076
## final  value 9570.675482 
## converged
## # weights:  160
## initial  value 13536.136100 
## iter  10 value 10075.946188
## iter  20 value 9913.960635
## iter  30 value 9638.939081
## iter  40 value 9511.042315
## iter  50 value 9459.452997
## iter  60 value 9400.474833
## iter  70 value 9352.490924
## iter  80 value 9245.438699
## iter  90 value 9179.894899
## iter 100 value 9137.915347
## final  value 9137.915347 
## stopped after 100 iterations
## # weights:  266
## initial  value 12192.161088 
## iter  10 value 9984.694877
## iter  20 value 9554.340300
## iter  30 value 9369.708214
## iter  40 value 9212.805379
## iter  50 value 9112.570592
## iter  60 value 9024.082602
## iter  70 value 8990.693691
## iter  80 value 8977.128964
## iter  90 value 8967.988762
## iter 100 value 8944.524816
## final  value 8944.524816 
## stopped after 100 iterations
## # weights:  54
## initial  value 12345.484532 
## iter  10 value 10955.335570
## iter  20 value 10149.151540
## iter  30 value 9864.042306
## iter  40 value 9671.861343
## iter  50 value 9604.692673
## iter  60 value 9582.085394
## iter  70 value 9580.380554
## iter  80 value 9580.371314
## iter  90 value 9580.302783
## iter 100 value 9579.023181
## final  value 9579.023181 
## stopped after 100 iterations
## # weights:  160
## initial  value 12546.139555 
## iter  10 value 9913.011659
## iter  20 value 9644.698863
## iter  30 value 9551.111372
## iter  40 value 9500.449834
## iter  50 value 9415.239850
## iter  60 value 9344.441913
## iter  70 value 9274.244956
## iter  80 value 9241.488835
## iter  90 value 9215.152105
## iter 100 value 9206.516657
## final  value 9206.516657 
## stopped after 100 iterations
## # weights:  266
## initial  value 12514.028271 
## iter  10 value 9858.073223
## iter  20 value 9573.621803
## iter  30 value 9372.159080
## iter  40 value 9256.783164
## iter  50 value 9192.411816
## iter  60 value 9064.156257
## iter  70 value 8953.706679
## iter  80 value 8843.254198
## iter  90 value 8789.354182
## iter 100 value 8710.894160
## final  value 8710.894160 
## stopped after 100 iterations
## # weights:  54
## initial  value 12005.836527 
## iter  10 value 11176.003038
## iter  20 value 10823.590457
## iter  30 value 10462.879188
## iter  40 value 10180.296951
## iter  50 value 9704.496176
## iter  60 value 9568.592762
## iter  70 value 9561.676478
## iter  80 value 9534.000020
## iter  90 value 9528.631458
## iter 100 value 9528.435304
## final  value 9528.435304 
## stopped after 100 iterations
## # weights:  160
## initial  value 14204.030809 
## iter  10 value 10477.156751
## iter  20 value 9757.126827
## iter  30 value 9591.341836
## iter  40 value 9458.423325
## iter  50 value 9375.267959
## iter  60 value 9330.737625
## iter  70 value 9304.607394
## iter  80 value 9282.819840
## iter  90 value 9274.530431
## iter 100 value 9271.452597
## final  value 9271.452597 
## stopped after 100 iterations
## # weights:  266
## initial  value 11990.464758 
## iter  10 value 9767.812089
## iter  20 value 9412.228872
## iter  30 value 9242.133072
## iter  40 value 9035.875897
## iter  50 value 8930.667398
## iter  60 value 8848.865829
## iter  70 value 8800.433318
## iter  80 value 8775.090683
## iter  90 value 8739.164842
## iter 100 value 8695.872833
## final  value 8695.872833 
## stopped after 100 iterations
## # weights:  54
## initial  value 12430.278310 
## iter  10 value 11671.679375
## iter  20 value 11373.272441
## iter  30 value 10802.144493
## iter  40 value 9858.514891
## iter  50 value 9661.485012
## iter  60 value 9631.681114
## iter  70 value 9565.719567
## iter  80 value 9549.775433
## iter  90 value 9542.542741
## iter 100 value 9541.192783
## final  value 9541.192783 
## stopped after 100 iterations
## # weights:  160
## initial  value 15732.558767 
## iter  10 value 10090.352269
## iter  20 value 9715.167636
## iter  30 value 9554.167055
## iter  40 value 9462.219157
## iter  50 value 9394.069817
## iter  60 value 9330.282221
## iter  70 value 9291.982386
## iter  80 value 9275.299177
## iter  90 value 9257.567694
## iter 100 value 9246.651981
## final  value 9246.651981 
## stopped after 100 iterations
## # weights:  266
## initial  value 13291.745786 
## iter  10 value 10055.896179
## iter  20 value 9569.356324
## iter  30 value 9381.789394
## iter  40 value 9178.126293
## iter  50 value 9077.752032
## iter  60 value 9030.104577
## iter  70 value 8972.877597
## iter  80 value 8899.554585
## iter  90 value 8804.927010
## iter 100 value 8692.543217
## final  value 8692.543217 
## stopped after 100 iterations
## # weights:  54
## initial  value 11981.571485 
## iter  10 value 10448.718881
## iter  20 value 9975.313499
## iter  30 value 9701.849204
## iter  40 value 9577.146866
## iter  50 value 9540.934639
## iter  60 value 9530.265992
## iter  70 value 9528.702723
## iter  80 value 9528.639029
## iter  80 value 9528.638978
## iter  80 value 9528.638975
## final  value 9528.638975 
## converged
## # weights:  160
## initial  value 18660.602873 
## iter  10 value 10072.576800
## iter  20 value 9686.687257
## iter  30 value 9469.342139
## iter  40 value 9355.110855
## iter  50 value 9282.305003
## iter  60 value 9210.597628
## iter  70 value 9144.007233
## iter  80 value 9121.468985
## iter  90 value 9108.849645
## iter 100 value 9040.092618
## final  value 9040.092618 
## stopped after 100 iterations
## # weights:  266
## initial  value 12520.707172 
## iter  10 value 9706.485452
## iter  20 value 9414.073029
## iter  30 value 9147.260006
## iter  40 value 8900.429153
## iter  50 value 8769.550934
## iter  60 value 8652.038223
## iter  70 value 8577.893951
## iter  80 value 8519.474439
## iter  90 value 8485.572231
## iter 100 value 8455.355813
## final  value 8455.355813 
## stopped after 100 iterations
## # weights:  54
## initial  value 12178.489860 
## iter  10 value 10958.318522
## iter  20 value 10310.553804
## iter  30 value 9944.673835
## iter  40 value 9883.681224
## iter  50 value 9806.889913
## iter  60 value 9752.261733
## iter  70 value 9738.980032
## iter  80 value 9738.524661
## iter  90 value 9738.422710
## iter 100 value 9738.337620
## final  value 9738.337620 
## stopped after 100 iterations
## # weights:  160
## initial  value 12264.454928 
## iter  10 value 10068.290616
## iter  20 value 9611.825261
## iter  30 value 9501.390057
## iter  40 value 9452.465920
## iter  50 value 9345.463621
## iter  60 value 9285.955931
## iter  70 value 9252.768853
## iter  80 value 9214.013112
## iter  90 value 9182.627467
## iter 100 value 9160.155312
## final  value 9160.155312 
## stopped after 100 iterations
## # weights:  266
## initial  value 12619.733785 
## iter  10 value 9835.504768
## iter  20 value 9435.078435
## iter  30 value 9267.671255
## iter  40 value 9156.999835
## iter  50 value 9031.053517
## iter  60 value 8883.208272
## iter  70 value 8829.184583
## iter  80 value 8790.199171
## iter  90 value 8727.241370
## iter 100 value 8687.184902
## final  value 8687.184902 
## stopped after 100 iterations
## # weights:  54
## initial  value 12572.962543 
## iter  10 value 11726.028436
## iter  20 value 10020.934363
## iter  30 value 9758.222097
## iter  40 value 9624.968042
## iter  50 value 9589.907910
## iter  60 value 9581.981104
## final  value 9580.523886 
## converged
## # weights:  160
## initial  value 14962.976782 
## iter  10 value 10279.108761
## iter  20 value 9686.270970
## iter  30 value 9536.038977
## iter  40 value 9407.424347
## iter  50 value 9324.889610
## iter  60 value 9290.709218
## iter  70 value 9273.604245
## iter  80 value 9267.541386
## iter  90 value 9264.625004
## iter 100 value 9262.163525
## final  value 9262.163525 
## stopped after 100 iterations
## # weights:  266
## initial  value 12498.803888 
## iter  10 value 9819.720999
## iter  20 value 9468.007628
## iter  30 value 9353.496584
## iter  40 value 9257.937303
## iter  50 value 9156.651871
## iter  60 value 9093.652438
## iter  70 value 9032.946694
## iter  80 value 8993.035879
## iter  90 value 8948.006025
## iter 100 value 8915.945544
## final  value 8915.945544 
## stopped after 100 iterations
## # weights:  54
## initial  value 12791.928776 
## final  value 11911.565361 
## converged
## # weights:  160
## initial  value 12720.901913 
## iter  10 value 10011.629520
## iter  20 value 9462.522779
## iter  30 value 9342.286341
## iter  40 value 9230.160187
## iter  50 value 9113.324463
## iter  60 value 9040.584764
## iter  70 value 8956.554074
## iter  80 value 8895.690323
## iter  90 value 8865.719518
## iter 100 value 8852.256525
## final  value 8852.256525 
## stopped after 100 iterations
## # weights:  266
## initial  value 11945.506081 
## iter  10 value 9973.536273
## iter  20 value 9539.884468
## iter  30 value 9273.751835
## iter  40 value 9120.288799
## iter  50 value 9030.163610
## iter  60 value 8979.755485
## iter  70 value 8900.062027
## iter  80 value 8837.957483
## iter  90 value 8772.523553
## iter 100 value 8707.893725
## final  value 8707.893725 
## stopped after 100 iterations
## # weights:  54
## initial  value 12178.579396 
## iter  10 value 11246.852597
## iter  20 value 10443.820834
## iter  30 value 9899.016510
## iter  40 value 9667.350843
## iter  50 value 9628.577941
## iter  60 value 9622.886181
## iter  70 value 9622.678563
## iter  80 value 9622.550593
## iter  90 value 9622.331384
## iter 100 value 9622.252698
## final  value 9622.252698 
## stopped after 100 iterations
## # weights:  160
## initial  value 12270.859478 
## iter  10 value 10302.686890
## iter  20 value 9640.314831
## iter  30 value 9468.067455
## iter  40 value 9385.290839
## iter  50 value 9323.834152
## iter  60 value 9292.520476
## iter  70 value 9224.362513
## iter  80 value 9157.779900
## iter  90 value 9120.183427
## iter 100 value 9098.741364
## final  value 9098.741364 
## stopped after 100 iterations
## # weights:  266
## initial  value 12271.504651 
## iter  10 value 10222.616202
## iter  20 value 9621.094883
## iter  30 value 9455.258342
## iter  40 value 9350.953902
## iter  50 value 9231.771498
## iter  60 value 9147.114190
## iter  70 value 9086.873632
## iter  80 value 9010.676302
## iter  90 value 8951.435385
## iter 100 value 8910.681657
## final  value 8910.681657 
## stopped after 100 iterations
## # weights:  54
## initial  value 12119.378681 
## iter  10 value 11832.462438
## iter  20 value 11391.108590
## iter  30 value 11136.606354
## iter  40 value 10530.260964
## iter  50 value 10199.131971
## iter  60 value 9837.655555
## iter  70 value 9714.496441
## iter  80 value 9648.003222
## iter  90 value 9612.216644
## iter 100 value 9596.882862
## final  value 9596.882862 
## stopped after 100 iterations
## # weights:  160
## initial  value 12677.584862 
## iter  10 value 9834.859445
## iter  20 value 9554.185109
## iter  30 value 9436.635368
## iter  40 value 9364.701957
## iter  50 value 9317.831702
## iter  60 value 9194.306731
## iter  70 value 9078.954951
## iter  80 value 9021.932491
## iter  90 value 8995.884652
## iter 100 value 8976.425277
## final  value 8976.425277 
## stopped after 100 iterations
## # weights:  266
## initial  value 14970.847230 
## iter  10 value 10193.610798
## iter  20 value 9597.820041
## iter  30 value 9392.187800
## iter  40 value 9246.162972
## iter  50 value 9152.848952
## iter  60 value 9096.648355
## iter  70 value 9033.065249
## iter  80 value 8944.536836
## iter  90 value 8867.938448
## iter 100 value 8832.791496
## final  value 8832.791496 
## stopped after 100 iterations
## # weights:  54
## initial  value 12071.651912 
## iter  10 value 11568.662344
## iter  20 value 11234.973376
## iter  30 value 10788.930493
## iter  40 value 10517.407075
## iter  50 value 10498.324362
## iter  60 value 10494.506047
## iter  70 value 10494.157607
## iter  80 value 10494.126457
## iter  80 value 10494.126354
## iter  80 value 10494.126329
## final  value 10494.126329 
## converged
## # weights:  160
## initial  value 11862.349456 
## iter  10 value 9791.118358
## iter  20 value 9559.381528
## iter  30 value 9384.152513
## iter  40 value 9306.521617
## iter  50 value 9268.474019
## iter  60 value 9232.411854
## iter  70 value 9203.707529
## iter  80 value 9173.206746
## iter  90 value 9149.784411
## iter 100 value 9129.071114
## final  value 9129.071114 
## stopped after 100 iterations
## # weights:  266
## initial  value 12265.422486 
## iter  10 value 9782.636950
## iter  20 value 9513.027931
## iter  30 value 9358.434189
## iter  40 value 9188.117037
## iter  50 value 9016.329705
## iter  60 value 8942.308547
## iter  70 value 8864.243089
## iter  80 value 8790.549268
## iter  90 value 8734.119158
## iter 100 value 8707.645483
## final  value 8707.645483 
## stopped after 100 iterations
## # weights:  54
## initial  value 12435.796388 
## iter  10 value 10562.454893
## iter  20 value 9861.580776
## iter  30 value 9753.839144
## iter  40 value 9729.036675
## iter  50 value 9715.498744
## iter  60 value 9702.904618
## iter  70 value 9701.991953
## final  value 9701.977664 
## converged
## # weights:  160
## initial  value 13627.199708 
## iter  10 value 10155.768081
## iter  20 value 9901.273067
## iter  30 value 9704.591540
## iter  40 value 9620.977992
## iter  50 value 9451.458459
## iter  60 value 9315.524812
## iter  70 value 9250.885853
## iter  80 value 9220.417156
## iter  90 value 9167.282288
## iter 100 value 9122.288918
## final  value 9122.288918 
## stopped after 100 iterations
## # weights:  266
## initial  value 11989.338981 
## iter  10 value 9819.941691
## iter  20 value 9604.082110
## iter  30 value 9460.560949
## iter  40 value 9338.162400
## iter  50 value 9255.800729
## iter  60 value 9134.545112
## iter  70 value 9029.617970
## iter  80 value 8933.829328
## iter  90 value 8847.447171
## iter 100 value 8753.239785
## final  value 8753.239785 
## stopped after 100 iterations
## # weights:  54
## initial  value 11803.276403 
## iter  10 value 11042.972260
## iter  20 value 10657.536953
## iter  30 value 10362.772590
## iter  40 value 10078.557127
## iter  50 value 9913.078880
## iter  60 value 9747.808761
## iter  70 value 9721.464596
## iter  80 value 9706.633977
## iter  90 value 9706.327450
## iter  90 value 9706.327418
## iter  90 value 9706.327418
## final  value 9706.327418 
## converged
## # weights:  160
## initial  value 12274.873387 
## iter  10 value 10116.201009
## iter  20 value 9684.254668
## iter  30 value 9493.153274
## iter  40 value 9439.035608
## iter  50 value 9406.357385
## iter  60 value 9380.481087
## iter  70 value 9361.637745
## iter  80 value 9355.182420
## iter  90 value 9352.988699
## iter 100 value 9350.736905
## final  value 9350.736905 
## stopped after 100 iterations
## # weights:  266
## initial  value 12374.602184 
## iter  10 value 9894.310821
## iter  20 value 9647.539335
## iter  30 value 9495.477465
## iter  40 value 9368.648255
## iter  50 value 9281.247548
## iter  60 value 9217.932138
## iter  70 value 9182.046877
## iter  80 value 9158.744593
## iter  90 value 9131.976459
## iter 100 value 9091.212596
## final  value 9091.212596 
## stopped after 100 iterations
## # weights:  54
## initial  value 12319.844795 
## iter  10 value 11508.872010
## iter  20 value 10005.731488
## iter  30 value 9738.187602
## iter  40 value 9721.610918
## iter  50 value 9712.956751
## iter  60 value 9704.570667
## iter  70 value 9702.065900
## final  value 9702.057857 
## converged
## # weights:  160
## initial  value 12274.339656 
## iter  10 value 9874.967051
## iter  20 value 9632.883689
## iter  30 value 9469.526003
## iter  40 value 9394.375461
## iter  50 value 9359.285976
## iter  60 value 9273.788772
## iter  70 value 9213.321252
## iter  80 value 9170.227856
## iter  90 value 9126.162470
## iter 100 value 9092.234839
## final  value 9092.234839 
## stopped after 100 iterations
## # weights:  266
## initial  value 12264.247688 
## iter  10 value 10089.224681
## iter  20 value 9577.948779
## iter  30 value 9466.390160
## iter  40 value 9312.613697
## iter  50 value 9199.762214
## iter  60 value 9036.386086
## iter  70 value 8934.684401
## iter  80 value 8882.775329
## iter  90 value 8859.437295
## iter 100 value 8836.174130
## final  value 8836.174130 
## stopped after 100 iterations
## # weights:  54
## initial  value 13566.662241 
## final  value 11909.867218 
## converged
## # weights:  160
## initial  value 12060.097570 
## iter  10 value 10077.522291
## iter  20 value 9575.562931
## iter  30 value 9482.467900
## iter  40 value 9393.542016
## iter  50 value 9272.969792
## iter  60 value 9211.085634
## iter  70 value 9186.340428
## iter  80 value 9161.110965
## iter  90 value 9136.658843
## iter 100 value 9115.226373
## final  value 9115.226373 
## stopped after 100 iterations
## # weights:  266
## initial  value 14700.889482 
## iter  10 value 9855.336802
## iter  20 value 9456.931591
## iter  30 value 9290.078519
## iter  40 value 9071.638152
## iter  50 value 8892.104026
## iter  60 value 8763.761413
## iter  70 value 8697.266563
## iter  80 value 8661.481792
## iter  90 value 8626.978629
## iter 100 value 8590.509100
## final  value 8590.509100 
## stopped after 100 iterations
## # weights:  54
## initial  value 12206.127503 
## iter  10 value 10927.864971
## iter  20 value 10626.594529
## iter  30 value 10435.704102
## iter  40 value 10336.528490
## iter  50 value 9880.009638
## iter  60 value 9619.644560
## iter  70 value 9576.677325
## iter  80 value 9564.334530
## iter  90 value 9554.324581
## iter 100 value 9551.176802
## final  value 9551.176802 
## stopped after 100 iterations
## # weights:  160
## initial  value 11715.950027 
## iter  10 value 9861.665330
## iter  20 value 9560.146930
## iter  30 value 9440.828138
## iter  40 value 9351.363816
## iter  50 value 9285.357158
## iter  60 value 9258.101217
## iter  70 value 9243.406074
## iter  80 value 9231.788761
## iter  90 value 9222.676761
## iter 100 value 9162.459190
## final  value 9162.459190 
## stopped after 100 iterations
## # weights:  266
## initial  value 12526.302286 
## iter  10 value 9883.860683
## iter  20 value 9481.967245
## iter  30 value 9351.417352
## iter  40 value 9238.643344
## iter  50 value 9172.121576
## iter  60 value 9132.457588
## iter  70 value 9107.616814
## iter  80 value 9077.299571
## iter  90 value 9056.557818
## iter 100 value 9042.725668
## final  value 9042.725668 
## stopped after 100 iterations
## # weights:  54
## initial  value 12492.879839 
## iter  10 value 10589.227112
## iter  20 value 9979.869692
## iter  30 value 9916.638276
## iter  40 value 9867.379501
## iter  50 value 9862.559550
## iter  60 value 9853.976836
## iter  70 value 9838.616040
## iter  80 value 9678.674100
## iter  90 value 9583.473863
## iter 100 value 9559.244525
## final  value 9559.244525 
## stopped after 100 iterations
## # weights:  160
## initial  value 12194.598800 
## iter  10 value 9903.749801
## iter  20 value 9809.696398
## iter  30 value 9645.170503
## iter  40 value 9550.822816
## iter  50 value 9480.266966
## iter  60 value 9381.577986
## iter  70 value 9323.822143
## iter  80 value 9281.654918
## iter  90 value 9262.000043
## iter 100 value 9237.079938
## final  value 9237.079938 
## stopped after 100 iterations
## # weights:  266
## initial  value 13045.609412 
## iter  10 value 9974.684105
## iter  20 value 9486.200011
## iter  30 value 9267.238927
## iter  40 value 9001.173742
## iter  50 value 8821.641530
## iter  60 value 8651.962478
## iter  70 value 8579.358818
## iter  80 value 8526.864489
## iter  90 value 8486.781868
## iter 100 value 8472.623138
## final  value 8472.623138 
## stopped after 100 iterations
## # weights:  54
## initial  value 12266.156780 
## iter  10 value 10150.015516
## iter  20 value 9852.452445
## iter  30 value 9656.842264
## iter  40 value 9616.773019
## iter  50 value 9604.953991
## iter  60 value 9592.077503
## iter  70 value 9591.750592
## iter  80 value 9591.714798
## iter  80 value 9591.714737
## iter  80 value 9591.714737
## final  value 9591.714737 
## converged
## # weights:  160
## initial  value 12814.783648 
## iter  10 value 10337.537900
## iter  20 value 9646.607998
## iter  30 value 9531.319395
## iter  40 value 9409.210478
## iter  50 value 9272.322587
## iter  60 value 9246.082266
## iter  70 value 9222.205478
## iter  80 value 9205.956256
## iter  90 value 9198.646597
## iter 100 value 9192.918050
## final  value 9192.918050 
## stopped after 100 iterations
## # weights:  266
## initial  value 12028.401507 
## iter  10 value 9792.008799
## iter  20 value 9415.854561
## iter  30 value 9214.222054
## iter  40 value 9128.361555
## iter  50 value 9054.579098
## iter  60 value 8966.229980
## iter  70 value 8931.135601
## iter  80 value 8903.370746
## iter  90 value 8878.747494
## iter 100 value 8840.268060
## final  value 8840.268060 
## stopped after 100 iterations
## # weights:  54
## initial  value 12196.195218 
## iter  10 value 11190.549670
## iter  20 value 10550.213341
## iter  30 value 10021.862523
## iter  40 value 9850.970441
## iter  50 value 9649.085179
## iter  60 value 9617.037581
## iter  70 value 9612.163631
## iter  80 value 9608.563958
## iter  90 value 9607.092660
## iter 100 value 9606.563875
## final  value 9606.563875 
## stopped after 100 iterations
## # weights:  160
## initial  value 12461.928347 
## iter  10 value 11348.257188
## iter  20 value 10180.101467
## iter  30 value 9867.596932
## iter  40 value 9650.975166
## iter  50 value 9466.534565
## iter  60 value 9383.304021
## iter  70 value 9333.736310
## iter  80 value 9314.955388
## iter  90 value 9303.836086
## iter 100 value 9276.860819
## final  value 9276.860819 
## stopped after 100 iterations
## # weights:  266
## initial  value 16358.093304 
## iter  10 value 10018.288505
## iter  20 value 9631.110791
## iter  30 value 9396.108121
## iter  40 value 9260.853454
## iter  50 value 9214.753985
## iter  60 value 9196.090489
## iter  70 value 9175.054216
## iter  80 value 9154.111829
## iter  90 value 9135.297703
## iter 100 value 9124.595390
## final  value 9124.595390 
## stopped after 100 iterations
## # weights:  54
## initial  value 12347.861751 
## iter  10 value 11704.839771
## iter  20 value 10998.631846
## iter  30 value 9998.689292
## iter  40 value 9726.902521
## iter  50 value 9676.251941
## iter  60 value 9672.702820
## iter  70 value 9672.545691
## iter  80 value 9672.490114
## iter  80 value 9672.490079
## iter  80 value 9672.490075
## final  value 9672.490075 
## converged
## # weights:  160
## initial  value 12013.591496 
## iter  10 value 10521.948731
## iter  20 value 9611.292390
## iter  30 value 9486.493526
## iter  40 value 9368.191990
## iter  50 value 9264.089510
## iter  60 value 9138.646358
## iter  70 value 9066.173689
## iter  80 value 9019.754163
## iter  90 value 8992.037167
## iter 100 value 8977.188120
## final  value 8977.188120 
## stopped after 100 iterations
## # weights:  266
## initial  value 12819.318609 
## iter  10 value 9816.407985
## iter  20 value 9398.761962
## iter  30 value 9258.218420
## iter  40 value 9108.278783
## iter  50 value 8974.661691
## iter  60 value 8880.095463
## iter  70 value 8799.155850
## iter  80 value 8704.726275
## iter  90 value 8653.999064
## iter 100 value 8608.278132
## final  value 8608.278132 
## stopped after 100 iterations
## # weights:  54
## initial  value 12534.092520 
## iter  10 value 11539.531203
## iter  20 value 10715.831057
## iter  30 value 10579.851906
## iter  40 value 10447.027013
## iter  50 value 10062.150779
## iter  60 value 9895.077470
## iter  70 value 9797.795127
## iter  80 value 9784.677757
## iter  90 value 9741.447830
## iter 100 value 9701.852842
## final  value 9701.852842 
## stopped after 100 iterations
## # weights:  160
## initial  value 14024.533485 
## iter  10 value 11399.430056
## iter  20 value 10240.423196
## iter  30 value 9661.476440
## iter  40 value 9554.400463
## iter  50 value 9493.235744
## iter  60 value 9469.302562
## iter  70 value 9395.272520
## iter  80 value 9329.629521
## iter  90 value 9277.162356
## iter 100 value 9237.344448
## final  value 9237.344448 
## stopped after 100 iterations
## # weights:  266
## initial  value 12021.103319 
## iter  10 value 10009.800653
## iter  20 value 9542.174020
## iter  30 value 9391.488903
## iter  40 value 9266.985766
## iter  50 value 9077.168638
## iter  60 value 8903.856421
## iter  70 value 8822.006246
## iter  80 value 8781.708621
## iter  90 value 8719.125218
## iter 100 value 8639.966461
## final  value 8639.966461 
## stopped after 100 iterations
## # weights:  54
## initial  value 12455.637244 
## iter  10 value 11912.606024
## iter  20 value 10260.147257
## iter  30 value 9788.165819
## iter  40 value 9707.616506
## iter  50 value 9649.950125
## iter  60 value 9622.304079
## iter  70 value 9618.496870
## final  value 9618.455591 
## converged
## # weights:  160
## initial  value 11864.119043 
## iter  10 value 10196.758702
## iter  20 value 9972.470922
## iter  30 value 9841.908163
## iter  40 value 9765.002373
## iter  50 value 9623.097102
## iter  60 value 9548.337160
## iter  70 value 9485.878408
## iter  80 value 9427.973599
## iter  90 value 9381.171076
## iter 100 value 9358.183757
## final  value 9358.183757 
## stopped after 100 iterations
## # weights:  266
## initial  value 11736.308608 
## iter  10 value 9907.131906
## iter  20 value 9480.868852
## iter  30 value 9344.294190
## iter  40 value 9228.208340
## iter  50 value 9153.405787
## iter  60 value 9094.470146
## iter  70 value 9067.775557
## iter  80 value 9058.400738
## iter  90 value 9051.712924
## iter 100 value 9042.829914
## final  value 9042.829914 
## stopped after 100 iterations
## # weights:  54
## initial  value 12415.732022 
## iter  10 value 11491.267795
## iter  20 value 10601.024087
## iter  30 value 10193.524567
## iter  40 value 9816.553563
## iter  50 value 9633.956312
## iter  60 value 9605.163236
## iter  70 value 9604.464568
## iter  80 value 9604.366736
## final  value 9604.358998 
## converged
## # weights:  160
## initial  value 12276.226795 
## iter  10 value 10080.237365
## iter  20 value 9846.707052
## iter  30 value 9664.707319
## iter  40 value 9570.268643
## iter  50 value 9500.719268
## iter  60 value 9425.351884
## iter  70 value 9365.620894
## iter  80 value 9324.310759
## iter  90 value 9294.448235
## iter 100 value 9224.390712
## final  value 9224.390712 
## stopped after 100 iterations
## # weights:  266
## initial  value 12325.587658 
## iter  10 value 9908.670341
## iter  20 value 9605.356071
## iter  30 value 9501.748980
## iter  40 value 9376.914653
## iter  50 value 9278.170514
## iter  60 value 9139.596102
## iter  70 value 9051.974215
## iter  80 value 8992.226533
## iter  90 value 8950.744192
## iter 100 value 8906.589894
## final  value 8906.589894 
## stopped after 100 iterations
## # weights:  54
## initial  value 12477.663776 
## iter  10 value 11691.096410
## iter  20 value 10682.438629
## iter  30 value 10228.907198
## iter  40 value 9817.503898
## iter  50 value 9694.827220
## iter  60 value 9680.531023
## iter  70 value 9668.041916
## final  value 9667.989863 
## converged
## # weights:  160
## initial  value 12140.883571 
## iter  10 value 10636.273498
## iter  20 value 9937.722322
## iter  30 value 9694.610307
## iter  40 value 9602.726778
## iter  50 value 9560.726075
## iter  60 value 9506.661435
## iter  70 value 9428.550638
## iter  80 value 9397.094244
## iter  90 value 9372.720404
## iter 100 value 9355.061515
## final  value 9355.061515 
## stopped after 100 iterations
## # weights:  266
## initial  value 12086.393982 
## iter  10 value 9955.054115
## iter  20 value 9583.759134
## iter  30 value 9379.653914
## iter  40 value 9176.661576
## iter  50 value 9002.957625
## iter  60 value 8810.399974
## iter  70 value 8671.014872
## iter  80 value 8609.644531
## iter  90 value 8584.975108
## iter 100 value 8569.604517
## final  value 8569.604517 
## stopped after 100 iterations
## # weights:  54
## initial  value 11974.729319 
## iter  10 value 10245.387841
## iter  20 value 9952.733682
## iter  30 value 9899.674165
## iter  40 value 9893.649806
## iter  50 value 9728.234093
## iter  60 value 9687.400922
## iter  70 value 9674.378898
## iter  80 value 9671.100086
## iter  90 value 9668.563515
## iter 100 value 9668.274522
## final  value 9668.274522 
## stopped after 100 iterations
## # weights:  160
## initial  value 11963.572040 
## iter  10 value 9891.784251
## iter  20 value 9714.453376
## iter  30 value 9632.501420
## iter  40 value 9523.617152
## iter  50 value 9391.933751
## iter  60 value 9327.670279
## iter  70 value 9303.389688
## iter  80 value 9294.446242
## iter  90 value 9290.108925
## iter 100 value 9288.389770
## final  value 9288.389770 
## stopped after 100 iterations
## # weights:  266
## initial  value 12811.954589 
## iter  10 value 10142.311272
## iter  20 value 9785.842797
## iter  30 value 9650.723062
## iter  40 value 9594.242614
## iter  50 value 9437.772869
## iter  60 value 9306.214654
## iter  70 value 9209.055030
## iter  80 value 9132.218062
## iter  90 value 9087.838890
## iter 100 value 9050.726085
## final  value 9050.726085 
## stopped after 100 iterations
## # weights:  54
## initial  value 12303.010283 
## iter  10 value 9864.272236
## iter  20 value 9760.766461
## iter  30 value 9699.521015
## iter  40 value 9678.338790
## iter  50 value 9665.683813
## iter  60 value 9653.437546
## iter  70 value 9653.224716
## final  value 9653.224517 
## converged
## # weights:  160
## initial  value 13056.767257 
## iter  10 value 10485.449889
## iter  20 value 9841.401054
## iter  30 value 9678.906427
## iter  40 value 9571.974581
## iter  50 value 9510.716908
## iter  60 value 9436.532021
## iter  70 value 9345.721586
## iter  80 value 9235.945809
## iter  90 value 9183.374200
## iter 100 value 9120.008492
## final  value 9120.008492 
## stopped after 100 iterations
## # weights:  266
## initial  value 13610.727139 
## iter  10 value 11157.799284
## iter  20 value 10178.038060
## iter  30 value 9641.889914
## iter  40 value 9495.346429
## iter  50 value 9393.129632
## iter  60 value 9215.734984
## iter  70 value 9049.866569
## iter  80 value 8902.558995
## iter  90 value 8793.094751
## iter 100 value 8728.413039
## final  value 8728.413039 
## stopped after 100 iterations
## # weights:  54
## initial  value 12216.254725 
## iter  10 value 10323.389224
## iter  20 value 9882.379135
## iter  30 value 9695.465437
## iter  40 value 9594.055478
## iter  50 value 9567.084420
## iter  60 value 9562.105817
## iter  70 value 9561.986525
## final  value 9561.979274 
## converged
## # weights:  160
## initial  value 12093.680035 
## iter  10 value 10606.727023
## iter  20 value 9865.465955
## iter  30 value 9523.722166
## iter  40 value 9396.783083
## iter  50 value 9279.627136
## iter  60 value 9188.971924
## iter  70 value 9122.662898
## iter  80 value 9095.015524
## iter  90 value 9070.406794
## iter 100 value 9032.994945
## final  value 9032.994945 
## stopped after 100 iterations
## # weights:  266
## initial  value 13663.293342 
## iter  10 value 9694.347879
## iter  20 value 9323.097749
## iter  30 value 9115.870545
## iter  40 value 8960.346992
## iter  50 value 8835.174372
## iter  60 value 8763.215235
## iter  70 value 8718.822506
## iter  80 value 8678.208658
## iter  90 value 8641.083830
## iter 100 value 8620.522939
## final  value 8620.522939 
## stopped after 100 iterations
## # weights:  54
## initial  value 14414.985329 
## iter  10 value 11921.475260
## iter  20 value 11134.615252
## iter  30 value 10373.781537
## iter  40 value 10198.373944
## iter  50 value 10130.436527
## iter  60 value 10121.382934
## iter  70 value 9892.253736
## iter  80 value 9677.047998
## iter  90 value 9591.172644
## iter 100 value 9565.644640
## final  value 9565.644640 
## stopped after 100 iterations
## # weights:  160
## initial  value 12256.398251 
## iter  10 value 10156.883441
## iter  20 value 9531.057253
## iter  30 value 9375.424043
## iter  40 value 9307.197856
## iter  50 value 9266.907729
## iter  60 value 9230.142471
## iter  70 value 9218.567842
## iter  80 value 9212.653429
## iter  90 value 9206.269774
## iter 100 value 9199.397927
## final  value 9199.397927 
## stopped after 100 iterations
## # weights:  266
## initial  value 11916.424568 
## iter  10 value 9773.059742
## iter  20 value 9580.456508
## iter  30 value 9500.899491
## iter  40 value 9416.246625
## iter  50 value 9333.131842
## iter  60 value 9263.770561
## iter  70 value 9222.162601
## iter  80 value 9178.031082
## iter  90 value 9149.634317
## iter 100 value 9126.060339
## final  value 9126.060339 
## stopped after 100 iterations
## # weights:  54
## initial  value 11991.199215 
## iter  10 value 10926.032829
## iter  20 value 10160.089304
## iter  30 value 9850.155675
## iter  40 value 9676.069750
## iter  50 value 9580.531704
## iter  60 value 9554.619748
## iter  70 value 9544.527325
## iter  80 value 9544.294263
## final  value 9544.290140 
## converged
## # weights:  160
## initial  value 13834.921233 
## iter  10 value 10440.748196
## iter  20 value 9967.540223
## iter  30 value 9823.925033
## iter  40 value 9788.302471
## iter  50 value 9746.047002
## iter  60 value 9638.350784
## iter  70 value 9476.354104
## iter  80 value 9431.075928
## iter  90 value 9392.330857
## iter 100 value 9347.219636
## final  value 9347.219636 
## stopped after 100 iterations
## # weights:  266
## initial  value 11969.949853 
## iter  10 value 10399.390299
## iter  20 value 9627.486919
## iter  30 value 9432.845312
## iter  40 value 9327.833655
## iter  50 value 9265.414829
## iter  60 value 9224.754308
## iter  70 value 9172.705407
## iter  80 value 9145.696558
## iter  90 value 9121.739547
## iter 100 value 9035.690508
## final  value 9035.690508 
## stopped after 100 iterations
## # weights:  54
## initial  value 12327.463481 
## iter  10 value 10885.473790
## iter  20 value 10442.259838
## iter  30 value 10255.636827
## iter  40 value 9948.995579
## iter  50 value 9910.145502
## iter  60 value 9898.530498
## iter  70 value 9882.199103
## iter  80 value 9832.280537
## iter  90 value 9792.385544
## iter 100 value 9789.191134
## final  value 9789.191134 
## stopped after 100 iterations
## # weights:  160
## initial  value 13457.270262 
## iter  10 value 9735.912137
## iter  20 value 9461.129374
## iter  30 value 9372.548083
## iter  40 value 9311.276149
## iter  50 value 9227.371630
## iter  60 value 9157.914314
## iter  70 value 9122.195653
## iter  80 value 9097.601827
## iter  90 value 9072.438154
## iter 100 value 9023.052821
## final  value 9023.052821 
## stopped after 100 iterations
## # weights:  266
## initial  value 11808.020759 
## iter  10 value 9817.959694
## iter  20 value 9331.643791
## iter  30 value 9153.170863
## iter  40 value 9059.150912
## iter  50 value 8928.160689
## iter  60 value 8839.951723
## iter  70 value 8733.704320
## iter  80 value 8650.156680
## iter  90 value 8579.460576
## iter 100 value 8521.522117
## final  value 8521.522117 
## stopped after 100 iterations
## # weights:  54
## initial  value 13105.555480 
## iter  10 value 10923.761024
## iter  20 value 10305.645597
## iter  30 value 10206.347681
## iter  40 value 10169.496632
## iter  50 value 10142.872992
## iter  60 value 10127.670044
## iter  70 value 10121.664800
## iter  80 value 9775.219216
## iter  90 value 9585.632967
## iter 100 value 9507.332826
## final  value 9507.332826 
## stopped after 100 iterations
## # weights:  160
## initial  value 12956.447813 
## iter  10 value 10444.821851
## iter  20 value 10088.973108
## iter  30 value 9933.425277
## iter  40 value 9846.196494
## iter  50 value 9810.638020
## iter  60 value 9670.828362
## iter  70 value 9479.285841
## iter  80 value 9387.813369
## iter  90 value 9324.719831
## iter 100 value 9233.587874
## final  value 9233.587874 
## stopped after 100 iterations
## # weights:  266
## initial  value 13446.880694 
## iter  10 value 9885.836438
## iter  20 value 9378.705996
## iter  30 value 9194.602406
## iter  40 value 9104.946138
## iter  50 value 9035.437438
## iter  60 value 8959.497827
## iter  70 value 8917.053293
## iter  80 value 8889.597127
## iter  90 value 8839.289369
## iter 100 value 8725.712074
## final  value 8725.712074 
## stopped after 100 iterations
## # weights:  54
## initial  value 12021.525121 
## iter  10 value 10908.568118
## iter  20 value 10836.126982
## iter  30 value 10389.854525
## iter  40 value 9969.609241
## iter  50 value 9786.490331
## iter  60 value 9729.682753
## iter  70 value 9727.543611
## iter  80 value 9726.209611
## iter  90 value 9725.283894
## iter 100 value 9725.220697
## final  value 9725.220697 
## stopped after 100 iterations
## # weights:  160
## initial  value 12186.433010 
## iter  10 value 10339.491954
## iter  20 value 9824.917050
## iter  30 value 9489.428317
## iter  40 value 9379.393790
## iter  50 value 9230.964244
## iter  60 value 9103.884936
## iter  70 value 9027.112033
## iter  80 value 8978.292401
## iter  90 value 8969.070603
## iter 100 value 8964.426617
## final  value 8964.426617 
## stopped after 100 iterations
## # weights:  266
## initial  value 13064.661627 
## iter  10 value 9992.025615
## iter  20 value 9412.717002
## iter  30 value 9210.569037
## iter  40 value 9099.141538
## iter  50 value 9001.354475
## iter  60 value 8951.126837
## iter  70 value 8915.433132
## iter  80 value 8881.257707
## iter  90 value 8863.127621
## iter 100 value 8843.822793
## final  value 8843.822793 
## stopped after 100 iterations
## # weights:  54
## initial  value 12755.864148 
## iter  10 value 10483.969308
## iter  20 value 10050.837836
## iter  30 value 10026.604315
## iter  40 value 9969.096788
## iter  50 value 9962.543145
## iter  60 value 9961.836982
## iter  70 value 9958.824817
## iter  80 value 9927.103960
## iter  90 value 9917.113734
## iter 100 value 9915.702197
## final  value 9915.702197 
## stopped after 100 iterations
## # weights:  160
## initial  value 12142.517697 
## iter  10 value 9790.140616
## iter  20 value 9581.560804
## iter  30 value 9445.163396
## iter  40 value 9374.147322
## iter  50 value 9282.562667
## iter  60 value 9189.903907
## iter  70 value 8983.607282
## iter  80 value 8909.155857
## iter  90 value 8876.932930
## iter 100 value 8860.952706
## final  value 8860.952706 
## stopped after 100 iterations
## # weights:  266
## initial  value 13519.495990 
## iter  10 value 9711.655287
## iter  20 value 9464.971509
## iter  30 value 9335.165293
## iter  40 value 9167.955593
## iter  50 value 9073.965412
## iter  60 value 9001.197697
## iter  70 value 8954.192578
## iter  80 value 8903.043323
## iter  90 value 8871.964055
## iter 100 value 8868.066026
## final  value 8868.066026 
## stopped after 100 iterations
## # weights:  54
## initial  value 12315.850330 
## iter  10 value 10357.941319
## iter  20 value 9849.801621
## iter  30 value 9666.607041
## iter  40 value 9581.548528
## iter  50 value 9564.956482
## iter  60 value 9560.660174
## final  value 9560.521830 
## converged
## # weights:  160
## initial  value 12707.251953 
## iter  10 value 9884.107608
## iter  20 value 9682.633117
## iter  30 value 9615.264707
## iter  40 value 9578.249198
## iter  50 value 9516.672505
## iter  60 value 9466.502928
## iter  70 value 9436.585392
## iter  80 value 9396.614000
## iter  90 value 9371.045404
## iter 100 value 9325.416715
## final  value 9325.416715 
## stopped after 100 iterations
## # weights:  266
## initial  value 12170.228127 
## iter  10 value 9965.273228
## iter  20 value 9543.540518
## iter  30 value 9445.364402
## iter  40 value 9343.331581
## iter  50 value 9244.358831
## iter  60 value 9193.038824
## iter  70 value 9166.432812
## iter  80 value 9136.221334
## iter  90 value 9077.311326
## iter 100 value 9004.092402
## final  value 9004.092402 
## stopped after 100 iterations
## # weights:  54
## initial  value 12526.665536 
## iter  10 value 10666.469530
## iter  20 value 10152.144164
## iter  30 value 9919.218377
## iter  40 value 9867.062616
## iter  50 value 9863.239847
## iter  60 value 9859.526230
## final  value 9859.502691 
## converged
## # weights:  160
## initial  value 12754.178663 
## iter  10 value 10037.764264
## iter  20 value 9628.767838
## iter  30 value 9462.225247
## iter  40 value 9424.798619
## iter  50 value 9401.026849
## iter  60 value 9385.259408
## iter  70 value 9348.296523
## iter  80 value 9329.852821
## iter  90 value 9322.948382
## iter 100 value 9318.953328
## final  value 9318.953328 
## stopped after 100 iterations
## # weights:  266
## initial  value 14283.685643 
## iter  10 value 10086.214881
## iter  20 value 9506.253641
## iter  30 value 9247.280851
## iter  40 value 9083.919366
## iter  50 value 8955.159370
## iter  60 value 8855.015717
## iter  70 value 8803.892482
## iter  80 value 8769.174500
## iter  90 value 8749.620223
## iter 100 value 8715.044405
## final  value 8715.044405 
## stopped after 100 iterations
## # weights:  54
## initial  value 12050.399032 
## iter  10 value 10351.415697
## iter  20 value 10105.965199
## iter  30 value 9948.518243
## iter  40 value 9885.603126
## iter  50 value 9876.509805
## iter  60 value 9875.998616
## iter  70 value 9875.871840
## iter  80 value 9870.242747
## iter  90 value 9696.434714
## iter 100 value 9649.142793
## final  value 9649.142793 
## stopped after 100 iterations
## # weights:  160
## initial  value 11861.452881 
## iter  10 value 10098.037877
## iter  20 value 9647.213844
## iter  30 value 9397.295967
## iter  40 value 9327.827431
## iter  50 value 9310.986471
## iter  60 value 9289.597209
## iter  70 value 9253.683286
## iter  80 value 9205.242299
## iter  90 value 9184.200035
## iter 100 value 9137.260282
## final  value 9137.260282 
## stopped after 100 iterations
## # weights:  266
## initial  value 14091.152363 
## iter  10 value 9855.310586
## iter  20 value 9507.912826
## iter  30 value 9296.093427
## iter  40 value 9087.619602
## iter  50 value 8902.904384
## iter  60 value 8721.452668
## iter  70 value 8575.882392
## iter  80 value 8499.940901
## iter  90 value 8443.138432
## iter 100 value 8417.682275
## final  value 8417.682275 
## stopped after 100 iterations
## # weights:  54
## initial  value 12034.637431 
## iter  10 value 10559.003314
## iter  20 value 10134.641360
## iter  30 value 9847.358684
## iter  40 value 9673.151507
## iter  50 value 9651.695303
## iter  60 value 9625.544748
## iter  70 value 9597.840353
## iter  80 value 9586.804469
## iter  90 value 9581.337244
## iter 100 value 9575.642452
## final  value 9575.642452 
## stopped after 100 iterations
## # weights:  160
## initial  value 12800.259050 
## iter  10 value 10018.758282
## iter  20 value 9618.397210
## iter  30 value 9520.027282
## iter  40 value 9400.399111
## iter  50 value 9346.167962
## iter  60 value 9317.345873
## iter  70 value 9286.928283
## iter  80 value 9267.048921
## iter  90 value 9249.460534
## iter 100 value 9225.439556
## final  value 9225.439556 
## stopped after 100 iterations
## # weights:  266
## initial  value 13684.238801 
## iter  10 value 9898.262345
## iter  20 value 9526.949229
## iter  30 value 9371.902263
## iter  40 value 9227.868548
## iter  50 value 9085.661800
## iter  60 value 8970.353129
## iter  70 value 8850.188822
## iter  80 value 8772.626900
## iter  90 value 8664.421632
## iter 100 value 8615.294875
## final  value 8615.294875 
## stopped after 100 iterations
## # weights:  54
## initial  value 13404.495343 
## iter  10 value 11606.215127
## iter  20 value 9926.187669
## iter  30 value 9685.082711
## iter  40 value 9634.742138
## iter  50 value 9584.214715
## iter  60 value 9565.611475
## iter  70 value 9564.929791
## iter  80 value 9564.887831
## iter  90 value 9564.877869
## iter  90 value 9564.877853
## iter  90 value 9564.877852
## final  value 9564.877852 
## converged
## # weights:  160
## initial  value 11944.855586 
## iter  10 value 9979.245558
## iter  20 value 9652.217542
## iter  30 value 9522.504408
## iter  40 value 9473.851083
## iter  50 value 9439.579342
## iter  60 value 9419.854686
## iter  70 value 9408.292225
## iter  80 value 9389.433174
## iter  90 value 9351.886318
## iter 100 value 9309.689789
## final  value 9309.689789 
## stopped after 100 iterations
## # weights:  266
## initial  value 14322.736234 
## iter  10 value 9990.451074
## iter  20 value 9527.153671
## iter  30 value 9402.606018
## iter  40 value 9342.596259
## iter  50 value 9304.022440
## iter  60 value 9217.405655
## iter  70 value 9146.289036
## iter  80 value 9007.091884
## iter  90 value 8930.525363
## iter 100 value 8853.159595
## final  value 8853.159595 
## stopped after 100 iterations
## # weights:  54
## initial  value 11947.530926 
## iter  10 value 10939.075691
## iter  20 value 10152.339109
## iter  30 value 9918.220334
## iter  40 value 9838.360182
## iter  50 value 9785.763309
## iter  60 value 9727.420610
## iter  70 value 9698.787578
## iter  80 value 9698.167179
## final  value 9698.165911 
## converged
## # weights:  160
## initial  value 12645.177988 
## iter  10 value 10691.033883
## iter  20 value 9954.686434
## iter  30 value 9719.686900
## iter  40 value 9606.186671
## iter  50 value 9581.449641
## iter  60 value 9574.274903
## iter  70 value 9567.637172
## iter  80 value 9552.613177
## iter  90 value 9525.383278
## iter 100 value 9489.879444
## final  value 9489.879444 
## stopped after 100 iterations
## # weights:  266
## initial  value 12628.716609 
## iter  10 value 10142.024530
## iter  20 value 9782.440628
## iter  30 value 9563.100417
## iter  40 value 9382.579861
## iter  50 value 9282.628018
## iter  60 value 9145.206283
## iter  70 value 9093.454712
## iter  80 value 9046.113574
## iter  90 value 9011.437498
## iter 100 value 8983.632947
## final  value 8983.632947 
## stopped after 100 iterations
## # weights:  54
## initial  value 11965.935641 
## iter  10 value 10945.629147
## iter  20 value 10597.793172
## iter  30 value 10307.274430
## iter  40 value 10009.561188
## iter  50 value 9800.737832
## iter  60 value 9695.451720
## iter  70 value 9671.161744
## iter  80 value 9670.653554
## final  value 9670.641822 
## converged
## # weights:  160
## initial  value 13066.178853 
## iter  10 value 11117.813436
## iter  20 value 10409.220854
## iter  30 value 10073.380290
## iter  40 value 9919.922161
## iter  50 value 9822.696687
## iter  60 value 9726.198780
## iter  70 value 9667.838456
## iter  80 value 9650.725616
## iter  90 value 9643.770939
## iter 100 value 9632.335875
## final  value 9632.335875 
## stopped after 100 iterations
## # weights:  266
## initial  value 12063.872977 
## iter  10 value 10029.494574
## iter  20 value 9633.797021
## iter  30 value 9454.485663
## iter  40 value 9349.442514
## iter  50 value 9290.760618
## iter  60 value 9262.048164
## iter  70 value 9246.649322
## iter  80 value 9224.421942
## iter  90 value 9202.341912
## iter 100 value 9192.412805
## final  value 9192.412805 
## stopped after 100 iterations
## # weights:  54
## initial  value 12195.269029 
## iter  10 value 10103.755816
## iter  20 value 9772.654156
## iter  30 value 9709.388210
## iter  40 value 9687.297677
## iter  50 value 9669.731276
## iter  60 value 9655.823303
## iter  70 value 9655.731742
## final  value 9655.730233 
## converged
## # weights:  160
## initial  value 13055.025607 
## iter  10 value 9910.521852
## iter  20 value 9658.526548
## iter  30 value 9525.443370
## iter  40 value 9461.040325
## iter  50 value 9394.833747
## iter  60 value 9300.992418
## iter  70 value 9220.399522
## iter  80 value 9169.402572
## iter  90 value 9149.755154
## iter 100 value 9131.162368
## final  value 9131.162368 
## stopped after 100 iterations
## # weights:  266
## initial  value 12734.804112 
## iter  10 value 10271.068374
## iter  20 value 9667.406717
## iter  30 value 9582.469219
## iter  40 value 9404.058234
## iter  50 value 9239.199469
## iter  60 value 9168.846197
## iter  70 value 9111.112604
## iter  80 value 9064.335008
## iter  90 value 9028.706539
## iter 100 value 8991.438805
## final  value 8991.438805 
## stopped after 100 iterations
## # weights:  54
## initial  value 12028.754835 
## iter  10 value 10628.735818
## iter  20 value 10010.049711
## iter  30 value 9748.407925
## iter  40 value 9679.848265
## iter  50 value 9668.167621
## iter  60 value 9664.825630
## iter  70 value 9664.813782
## iter  80 value 9664.164891
## iter  90 value 9662.843911
## iter 100 value 9661.985438
## final  value 9661.985438 
## stopped after 100 iterations
## # weights:  160
## initial  value 12297.925972 
## iter  10 value 10135.112450
## iter  20 value 9780.904552
## iter  30 value 9580.128955
## iter  40 value 9449.438020
## iter  50 value 9363.064920
## iter  60 value 9342.659563
## iter  70 value 9330.843151
## iter  80 value 9310.632829
## iter  90 value 9299.357001
## iter 100 value 9283.267812
## final  value 9283.267812 
## stopped after 100 iterations
## # weights:  266
## initial  value 12661.369402 
## iter  10 value 9882.979945
## iter  20 value 9591.456693
## iter  30 value 9431.742572
## iter  40 value 9293.369974
## iter  50 value 9169.854336
## iter  60 value 8991.133005
## iter  70 value 8868.712335
## iter  80 value 8805.270186
## iter  90 value 8766.769044
## iter 100 value 8748.422287
## final  value 8748.422287 
## stopped after 100 iterations
## # weights:  54
## initial  value 11906.566419 
## iter  10 value 10142.185236
## iter  20 value 9882.196409
## iter  30 value 9777.420711
## iter  40 value 9695.502211
## iter  50 value 9685.639345
## iter  60 value 9680.006512
## iter  70 value 9676.640207
## iter  80 value 9675.264617
## iter  90 value 9674.653729
## final  value 9674.606618 
## converged
## # weights:  160
## initial  value 12379.346886 
## iter  10 value 10072.976750
## iter  20 value 9595.322705
## iter  30 value 9464.558110
## iter  40 value 9410.955196
## iter  50 value 9373.484994
## iter  60 value 9347.013517
## iter  70 value 9335.234047
## iter  80 value 9329.446727
## iter  90 value 9325.151403
## iter 100 value 9322.213141
## final  value 9322.213141 
## stopped after 100 iterations
## # weights:  266
## initial  value 12026.112326 
## iter  10 value 9838.184858
## iter  20 value 9589.922712
## iter  30 value 9426.238587
## iter  40 value 9300.870658
## iter  50 value 9197.643134
## iter  60 value 9031.837449
## iter  70 value 8905.511427
## iter  80 value 8832.974154
## iter  90 value 8788.262112
## iter 100 value 8755.310586
## final  value 8755.310586 
## stopped after 100 iterations
## # weights:  54
## initial  value 13795.642366 
## iter  10 value 11184.904135
## iter  20 value 11013.577749
## iter  30 value 10987.014918
## iter  40 value 10983.270064
## iter  50 value 10965.529848
## iter  60 value 10953.607535
## iter  70 value 10952.580643
## iter  80 value 10950.914108
## iter  90 value 10950.051838
## iter 100 value 10945.720117
## final  value 10945.720117 
## stopped after 100 iterations
## # weights:  160
## initial  value 11843.057169 
## iter  10 value 10121.946970
## iter  20 value 9701.775719
## iter  30 value 9602.189431
## iter  40 value 9543.764419
## iter  50 value 9445.345666
## iter  60 value 9310.108743
## iter  70 value 9245.596393
## iter  80 value 9184.372589
## iter  90 value 9140.356889
## iter 100 value 9077.234309
## final  value 9077.234309 
## stopped after 100 iterations
## # weights:  266
## initial  value 13337.233696 
## iter  10 value 10954.426488
## iter  20 value 10210.259375
## iter  30 value 9623.711807
## iter  40 value 9359.221866
## iter  50 value 9252.658714
## iter  60 value 9134.197247
## iter  70 value 9072.684546
## iter  80 value 9027.661889
## iter  90 value 9001.706876
## iter 100 value 8969.367483
## final  value 8969.367483 
## stopped after 100 iterations
## # weights:  54
## initial  value 12356.869682 
## iter  10 value 10961.074520
## iter  20 value 10510.095712
## iter  30 value 10123.130506
## iter  40 value 9985.020968
## iter  50 value 9965.713292
## iter  60 value 9931.943563
## iter  70 value 9789.208573
## iter  80 value 9729.171601
## iter  90 value 9706.052523
## iter 100 value 9700.782129
## final  value 9700.782129 
## stopped after 100 iterations
## # weights:  160
## initial  value 12169.810227 
## iter  10 value 10814.757881
## iter  20 value 9847.007071
## iter  30 value 9601.751542
## iter  40 value 9481.255594
## iter  50 value 9436.270079
## iter  60 value 9413.821797
## iter  70 value 9382.306568
## iter  80 value 9306.424765
## iter  90 value 9253.853569
## iter 100 value 9240.748824
## final  value 9240.748824 
## stopped after 100 iterations
## # weights:  266
## initial  value 11992.766434 
## iter  10 value 9852.488142
## iter  20 value 9532.016910
## iter  30 value 9384.156861
## iter  40 value 9243.573131
## iter  50 value 9131.489072
## iter  60 value 9056.358459
## iter  70 value 8976.871459
## iter  80 value 8911.646609
## iter  90 value 8841.191366
## iter 100 value 8798.989254
## final  value 8798.989254 
## stopped after 100 iterations
## # weights:  54
## initial  value 12962.960709 
## iter  10 value 10178.363777
## iter  20 value 9844.624432
## iter  30 value 9769.297631
## iter  40 value 9738.958920
## iter  50 value 9718.330115
## iter  60 value 9711.439443
## iter  70 value 9710.504618
## final  value 9710.496037 
## converged
## # weights:  160
## initial  value 12250.493918 
## iter  10 value 10161.975482
## iter  20 value 9892.164143
## iter  30 value 9804.421773
## iter  40 value 9737.459435
## iter  50 value 9691.939926
## iter  60 value 9620.260550
## iter  70 value 9573.018000
## iter  80 value 9555.180798
## iter  90 value 9535.951845
## iter 100 value 9517.198271
## final  value 9517.198271 
## stopped after 100 iterations
## # weights:  266
## initial  value 12617.825745 
## iter  10 value 9874.213265
## iter  20 value 9539.627470
## iter  30 value 9400.844375
## iter  40 value 9336.057708
## iter  50 value 9292.038830
## iter  60 value 9246.418940
## iter  70 value 9182.607052
## iter  80 value 9122.676508
## iter  90 value 9061.589181
## iter 100 value 9032.152608
## final  value 9032.152608 
## stopped after 100 iterations
## # weights:  54
## initial  value 12180.616021 
## iter  10 value 10151.212778
## iter  20 value 9996.306578
## iter  30 value 9933.723467
## iter  40 value 9758.986133
## iter  50 value 9729.685971
## iter  60 value 9714.622528
## iter  70 value 9711.200976
## iter  80 value 9704.840853
## iter  90 value 9701.709186
## iter 100 value 9700.655320
## final  value 9700.655320 
## stopped after 100 iterations
## # weights:  160
## initial  value 14907.359449 
## iter  10 value 10033.344136
## iter  20 value 9668.976500
## iter  30 value 9485.324989
## iter  40 value 9432.083002
## iter  50 value 9380.242051
## iter  60 value 9316.285675
## iter  70 value 9222.615150
## iter  80 value 9138.237053
## iter  90 value 9105.553521
## iter 100 value 9090.029026
## final  value 9090.029026 
## stopped after 100 iterations
## # weights:  266
## initial  value 14503.181545 
## iter  10 value 9866.121315
## iter  20 value 9604.414890
## iter  30 value 9445.959326
## iter  40 value 9311.584873
## iter  50 value 9205.961124
## iter  60 value 9154.881027
## iter  70 value 9092.061700
## iter  80 value 8998.948579
## iter  90 value 8933.575261
## iter 100 value 8842.151120
## final  value 8842.151120 
## stopped after 100 iterations
## # weights:  54
## initial  value 12566.694784 
## iter  10 value 10871.019194
## iter  20 value 10345.222332
## iter  30 value 10045.868884
## iter  40 value 9678.123288
## iter  50 value 9606.515592
## iter  60 value 9585.002051
## iter  70 value 9579.354088
## iter  80 value 9578.826115
## iter  90 value 9578.748925
## iter 100 value 9578.690767
## final  value 9578.690767 
## stopped after 100 iterations
## # weights:  160
## initial  value 12013.144320 
## iter  10 value 10491.289118
## iter  20 value 9890.507768
## iter  30 value 9573.804032
## iter  40 value 9449.134445
## iter  50 value 9387.767222
## iter  60 value 9351.489114
## iter  70 value 9307.925274
## iter  80 value 9285.698576
## iter  90 value 9254.531468
## iter 100 value 9236.390395
## final  value 9236.390395 
## stopped after 100 iterations
## # weights:  266
## initial  value 12119.240266 
## iter  10 value 9814.962200
## iter  20 value 9541.707509
## iter  30 value 9356.666751
## iter  40 value 9252.323837
## iter  50 value 9145.862856
## iter  60 value 8964.289144
## iter  70 value 8813.060554
## iter  80 value 8709.992953
## iter  90 value 8665.933902
## iter 100 value 8642.016262
## final  value 8642.016262 
## stopped after 100 iterations
## # weights:  54
## initial  value 12803.362655 
## iter  10 value 11504.360685
## iter  20 value 10798.009753
## iter  30 value 10040.798491
## iter  40 value 9788.342783
## iter  50 value 9681.426545
## iter  60 value 9608.061554
## iter  70 value 9592.064548
## iter  80 value 9590.524774
## final  value 9590.496658 
## converged
## # weights:  160
## initial  value 11932.928138 
## iter  10 value 10559.413369
## iter  20 value 9817.238623
## iter  30 value 9699.250954
## iter  40 value 9642.128316
## iter  50 value 9581.809933
## iter  60 value 9534.426165
## iter  70 value 9475.494335
## iter  80 value 9441.096378
## iter  90 value 9419.172422
## iter 100 value 9395.832507
## final  value 9395.832507 
## stopped after 100 iterations
## # weights:  266
## initial  value 14448.924729 
## iter  10 value 10644.967130
## iter  20 value 9909.804934
## iter  30 value 9496.516555
## iter  40 value 9405.733643
## iter  50 value 9310.824662
## iter  60 value 9185.653149
## iter  70 value 9109.587995
## iter  80 value 9014.244921
## iter  90 value 8900.988499
## iter 100 value 8806.825326
## final  value 8806.825326 
## stopped after 100 iterations
## # weights:  54
## initial  value 13055.460461 
## iter  10 value 11289.409062
## iter  20 value 10445.335570
## iter  30 value 10267.872339
## iter  40 value 10048.102442
## iter  50 value 9768.692204
## iter  60 value 9600.721920
## iter  70 value 9580.052865
## iter  80 value 9578.900193
## iter  90 value 9578.696578
## final  value 9578.696103 
## converged
## # weights:  160
## initial  value 12286.675058 
## iter  10 value 10467.908648
## iter  20 value 10170.102958
## iter  30 value 9801.381156
## iter  40 value 9686.974245
## iter  50 value 9589.618257
## iter  60 value 9545.794404
## iter  70 value 9490.657830
## iter  80 value 9470.025905
## iter  90 value 9457.086757
## iter 100 value 9451.301378
## final  value 9451.301378 
## stopped after 100 iterations
## # weights:  266
## initial  value 12040.496125 
## iter  10 value 9773.285784
## iter  20 value 9554.804141
## iter  30 value 9272.247528
## iter  40 value 9150.077969
## iter  50 value 9027.889594
## iter  60 value 8935.145069
## iter  70 value 8881.663666
## iter  80 value 8843.933165
## iter  90 value 8785.076432
## iter 100 value 8729.796287
## final  value 8729.796287 
## stopped after 100 iterations
## # weights:  54
## initial  value 13246.104344 
## iter  10 value 10446.363145
## iter  20 value 10150.809974
## iter  30 value 9843.301537
## iter  40 value 9654.717223
## iter  50 value 9514.023514
## iter  60 value 9503.519301
## iter  70 value 9502.553638
## iter  80 value 9502.536831
## final  value 9502.531561 
## converged
## # weights:  160
## initial  value 12171.778088 
## iter  10 value 9898.511447
## iter  20 value 9468.737707
## iter  30 value 9333.776443
## iter  40 value 9228.935330
## iter  50 value 9158.842495
## iter  60 value 9140.231877
## iter  70 value 9131.548905
## iter  80 value 9123.593750
## iter  90 value 9114.061515
## iter 100 value 9057.400254
## final  value 9057.400254 
## stopped after 100 iterations
## # weights:  266
## initial  value 12121.522210 
## iter  10 value 10197.448937
## iter  20 value 9540.725465
## iter  30 value 9367.648046
## iter  40 value 9201.809811
## iter  50 value 8980.832721
## iter  60 value 8870.298428
## iter  70 value 8751.404307
## iter  80 value 8668.244164
## iter  90 value 8633.809655
## iter 100 value 8611.679434
## final  value 8611.679434 
## stopped after 100 iterations
## # weights:  54
## initial  value 11926.083773 
## iter  10 value 10224.069528
## iter  20 value 9707.191148
## iter  30 value 9630.472124
## iter  40 value 9583.817548
## iter  50 value 9562.370329
## iter  60 value 9517.211684
## iter  70 value 9515.127861
## iter  80 value 9514.891396
## final  value 9514.880562 
## converged
## # weights:  160
## initial  value 12092.860716 
## iter  10 value 10037.449567
## iter  20 value 9661.851396
## iter  30 value 9548.220790
## iter  40 value 9513.233153
## iter  50 value 9445.480018
## iter  60 value 9368.896737
## iter  70 value 9289.865282
## iter  80 value 9249.810240
## iter  90 value 9219.530736
## iter 100 value 9189.144636
## final  value 9189.144636 
## stopped after 100 iterations
## # weights:  266
## initial  value 12606.940351 
## iter  10 value 9922.293446
## iter  20 value 9504.544501
## iter  30 value 9401.123134
## iter  40 value 9264.698079
## iter  50 value 9160.210731
## iter  60 value 9042.607257
## iter  70 value 8955.726411
## iter  80 value 8882.764013
## iter  90 value 8858.363549
## iter 100 value 8837.612203
## final  value 8837.612203 
## stopped after 100 iterations
## # weights:  54
## initial  value 12602.683065 
## iter  10 value 10231.727017
## iter  20 value 9835.864911
## iter  30 value 9630.648871
## iter  40 value 9576.085416
## iter  50 value 9553.974328
## iter  60 value 9546.552921
## iter  70 value 9529.185485
## iter  80 value 9512.278757
## iter  90 value 9504.714430
## iter 100 value 9504.005369
## final  value 9504.005369 
## stopped after 100 iterations
## # weights:  160
## initial  value 12208.813560 
## iter  10 value 10306.779540
## iter  20 value 9604.582889
## iter  30 value 9403.370032
## iter  40 value 9322.069653
## iter  50 value 9235.963657
## iter  60 value 9190.811908
## iter  70 value 9158.895365
## iter  80 value 9140.476061
## iter  90 value 9119.050847
## iter 100 value 9102.945892
## final  value 9102.945892 
## stopped after 100 iterations
## # weights:  266
## initial  value 12654.655476 
## iter  10 value 9605.385213
## iter  20 value 9403.499508
## iter  30 value 9279.829881
## iter  40 value 9134.492687
## iter  50 value 8996.562247
## iter  60 value 8828.218968
## iter  70 value 8713.252993
## iter  80 value 8670.568031
## iter  90 value 8634.011269
## iter 100 value 8598.763082
## final  value 8598.763082 
## stopped after 100 iterations
## # weights:  54
## initial  value 11945.272486 
## iter  10 value 10161.647646
## iter  20 value 9851.805036
## iter  30 value 9676.779374
## iter  40 value 9620.319777
## iter  50 value 9609.314860
## iter  60 value 9605.546994
## iter  70 value 9604.963407
## iter  80 value 9602.765654
## iter  90 value 9602.321496
## iter 100 value 9598.354768
## final  value 9598.354768 
## stopped after 100 iterations
## # weights:  160
## initial  value 12113.321785 
## iter  10 value 10537.242366
## iter  20 value 10009.468863
## iter  30 value 9770.699490
## iter  40 value 9634.769643
## iter  50 value 9584.474729
## iter  60 value 9557.364877
## iter  70 value 9531.159698
## iter  80 value 9500.663935
## iter  90 value 9473.960717
## iter 100 value 9458.051051
## final  value 9458.051051 
## stopped after 100 iterations
## # weights:  266
## initial  value 12492.574044 
## iter  10 value 9766.754339
## iter  20 value 9517.407992
## iter  30 value 9314.836319
## iter  40 value 9195.891180
## iter  50 value 9056.453100
## iter  60 value 8912.470785
## iter  70 value 8797.940983
## iter  80 value 8689.659435
## iter  90 value 8639.477456
## iter 100 value 8607.909882
## final  value 8607.909882 
## stopped after 100 iterations
## # weights:  54
## initial  value 12616.607053 
## iter  10 value 11029.794630
## iter  20 value 10894.312153
## iter  30 value 10239.952428
## iter  40 value 9831.281522
## iter  50 value 9708.052433
## iter  60 value 9657.861654
## iter  70 value 9636.365999
## iter  80 value 9634.831856
## iter  90 value 9620.570320
## iter 100 value 9600.285230
## final  value 9600.285230 
## stopped after 100 iterations
## # weights:  160
## initial  value 12314.667905 
## iter  10 value 9977.432770
## iter  20 value 9663.216150
## iter  30 value 9569.013090
## iter  40 value 9507.925719
## iter  50 value 9399.448912
## iter  60 value 9265.723499
## iter  70 value 9202.249754
## iter  80 value 9170.511503
## iter  90 value 9149.385612
## iter 100 value 9132.258976
## final  value 9132.258976 
## stopped after 100 iterations
## # weights:  266
## initial  value 12522.236462 
## iter  10 value 9829.975936
## iter  20 value 9539.160367
## iter  30 value 9395.090419
## iter  40 value 9254.218793
## iter  50 value 9108.594077
## iter  60 value 8963.883021
## iter  70 value 8835.163982
## iter  80 value 8749.132883
## iter  90 value 8700.776771
## iter 100 value 8649.315102
## final  value 8649.315102 
## stopped after 100 iterations
## # weights:  54
## initial  value 12314.154169 
## iter  10 value 11495.516481
## iter  20 value 10852.901223
## iter  30 value 10369.354112
## iter  40 value 10065.624633
## iter  50 value 9882.989771
## iter  60 value 9692.899286
## iter  70 value 9644.972731
## iter  80 value 9633.301886
## iter  90 value 9579.674463
## iter 100 value 9569.309225
## final  value 9569.309225 
## stopped after 100 iterations
## # weights:  160
## initial  value 11968.394656 
## iter  10 value 9916.472034
## iter  20 value 9615.418382
## iter  30 value 9381.524651
## iter  40 value 9239.414910
## iter  50 value 9148.250287
## iter  60 value 9099.490656
## iter  70 value 9068.073956
## iter  80 value 9040.548270
## iter  90 value 9021.399748
## iter 100 value 9006.454440
## final  value 9006.454440 
## stopped after 100 iterations
## # weights:  266
## initial  value 12047.461868 
## iter  10 value 9826.125424
## iter  20 value 9444.714466
## iter  30 value 9259.602060
## iter  40 value 9120.881443
## iter  50 value 9018.668435
## iter  60 value 8976.338233
## iter  70 value 8932.370342
## iter  80 value 8887.928945
## iter  90 value 8848.165025
## iter 100 value 8792.607256
## final  value 8792.607256 
## stopped after 100 iterations
## # weights:  266
## initial  value 12267.069806 
## iter  10 value 9774.232819
## iter  20 value 9459.040194
## iter  30 value 9318.133942
## iter  40 value 9268.860428
## iter  50 value 9228.161882
## iter  60 value 9174.827456
## iter  70 value 9104.459657
## iter  80 value 8985.980379
## iter  90 value 8854.118870
## iter 100 value 8746.288603
## final  value 8746.288603 
## stopped after 100 iterations

print(nn)

## Neural Network 
## 
## 17185 samples
##    51 predictor
##     2 classes: '0', '1' 
## 
## No pre-processing
## Resampling: Bootstrapped (25 reps) 
## Summary of sample sizes: 17185, 17185, 17185, 17185, 17185, 17185, ... 
## Resampling results across tuning parameters:
## 
##   size  decay  Accuracy   Kappa    
##   1     0e+00  0.7012950  0.4040463
##   1     1e-04  0.6980177  0.3976095
##   1     1e-01  0.7141634  0.4293829
##   3     0e+00  0.7163210  0.4337046
##   3     1e-04  0.7236862  0.4482863
##   3     1e-01  0.7214693  0.4438824
##   5     0e+00  0.7242442  0.4489788
##   5     1e-04  0.7222741  0.4454377
##   5     1e-01  0.7321665  0.4649093
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 5 and decay = 0.1.

plotnet(nn)

#variable importance(garson belongs to NeuralNetTools)

garson(nn$finalModel)+theme(axis.text.x     = element_text(angle = 45, hjust = 1, size=10))

#train
nnpredtrain <- predict(nn, trainNN)
resulttrainnn <-table(predicted=nnpredtrain,true=trainNN$y)
resulttrainnn

##          true
## predicted    0    1
##         0 7134 2565
##         1 1399 6087

acctrainnn =sum(diag(resulttrainnn))/sum(resulttrainnn)
acctrainnn

## [1] 0.7693337

#test
nnpredtest <- predict(nn, testNN)
resulttestnn <-table(predicted=nnpredtest,true=testNN$y)
resulttestnn

##          true
## predicted    0    1
##         0 7184 2609
##         1 1536 5856

acctestnn =sum(diag(resulttestnn))/sum(resulttestnn)
acctestnn

## [1] 0.7588013

png("nn.png",height=2500, width=3000) 
plot(nn) 
dev.off()

## quartz_off_screen 
##                 2

#=============================================================